From c4bfecd1555b533cbb95818c887b05886ad8c01d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 16 Oct 2026 18:13:05 -0400 Subject: [PATCH] Add customer-controlled experimentation opt-out --- .../openfeature/OpenFeatureConfiguration.java | 6 +- .../OpenFeatureProviderSmokeTest.groovy | 72 +++ .../src/test/resources/ffe-system-test-data | 2 +- .../feature-flagging-api/README.md | 34 ++ .../trace/api/openfeature/DDEvaluator.java | 54 +- .../trace/api/openfeature/Evaluator.java | 19 + .../trace/api/openfeature/Provider.java | 45 +- .../EvaluationTelemetryOptOutTest.java | 115 +++- .../ExperimentationOptOutForkedTest.java | 513 ++++++++++++++++++ .../ufc/v1/ServerConfiguration.java | 15 + .../JsonApiUfcResponseParserTest.java | 16 + 11 files changed, 882 insertions(+), 9 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ExperimentationOptOutForkedTest.java diff --git a/dd-smoke-tests/openfeature/application/src/main/java/datadog/smoketest/springboot/openfeature/OpenFeatureConfiguration.java b/dd-smoke-tests/openfeature/application/src/main/java/datadog/smoketest/springboot/openfeature/OpenFeatureConfiguration.java index f8a8ac51e45..b7b55888fa1 100644 --- a/dd-smoke-tests/openfeature/application/src/main/java/datadog/smoketest/springboot/openfeature/OpenFeatureConfiguration.java +++ b/dd-smoke-tests/openfeature/application/src/main/java/datadog/smoketest/springboot/openfeature/OpenFeatureConfiguration.java @@ -12,7 +12,11 @@ public class OpenFeatureConfiguration { @Bean public Client openFeatureClient() { OpenFeatureAPI api = OpenFeatureAPI.getInstance(); - api.setProviderAndWait(new Provider()); + api.setProviderAndWait( + new Provider( + new Provider.Options() + .enableExperimentationOptOutParameter( + Boolean.getBoolean("smoketest.experimentationOptOutParameterEnabled")))); return api.getClient(); } } diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy index 12115cb407d..9d5759f9f51 100644 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy +++ b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy @@ -38,6 +38,7 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { command.addAll(defaultJavaProperties) command.add('-Ddd.trace.debug=true') command.add('-Ddd.remote_config.enabled=true') + command.add('-Dsmoketest.experimentationOptOutParameterEnabled=true') command.add("-Ddd.remote_config.url=http://localhost:${server.address.port}/v0.7/config".toString()) command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) @@ -179,6 +180,77 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { testCase << parseTestCases() } + @Unroll("test experimentation opt-out contract - #testCase.id") + void 'test experimentation opt-out through the provider and agent'() { + setup: + final configuration = new JsonSlurper().parse(fetchResource( + 'ffe-system-test-data/experimentation-opt-out/v1/configurations/supported.json')) + setRemoteConfig('datadog/2/FFE_FLAGS/1/config', JsonOutput.toJson(configuration)) + final request = new Request.Builder() + .url("http://localhost:${httpPort}/openfeature/evaluate") + .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson([ + flag: testCase.evaluation.flag, + variationType: testCase.evaluation.variationType, + defaultValue: testCase.evaluation.defaultValue, + targetingKey: testCase.context.targetingKey, + attributes: testCase.context.attributes + ]))) + .build() + + expect: + // Wait for Remote Configuration delivery. These fixtures have no time-dependent allocations. + new PollingConditions(timeout: 10).eventually { + final response = client.newCall(request).execute() + try { + assert response.code() == 200 + final body = new JsonSlurper().parse(response.body().byteStream()) + assert body.value == testCase.expected.value + assert body.errorCode == null + final assignment = testCase.expected.assignment + if (assignment == null) { + assert body.flagMetadata?.allocationKey == null + assert body.reason == 'DEFAULT' + } else { + assert body.variant == assignment.variationKey + assert body.flagMetadata.allocationKey == assignment.allocationKey + assert body.flagMetadata.__dd_split_serial_id == assignment.serialId + assert body.flagMetadata.__dd_do_log == assignment.doLog + final prefixes = [HOOK: '__dd_feature.', EXPOSURE: '__dd_exposure_feature.', EVALUATION: '__dd_evaluation_feature.'] + final expectedFeatures = expectedFeatureMetadata(assignment, prefixes) + assert body.flagMetadata.findAll { key, value -> + prefixes.values().any { key.startsWith(it) } + } == expectedFeatures + } + } finally { + response.close() + } + } + + where: + testCase << parseExperimentationOptOutCases() + } + + private static List> parseExperimentationOptOutCases() { + final slurper = new JsonSlurper() + final cases = ['activation', 'evaluation', 'holdouts'].collectMany { name -> + slurper.parse(fetchResource("ffe-system-test-data/experimentation-opt-out/v1/cases/${name}.json")) + }.findAll { + it.configuration == 'supported' && it.providerOptions.enableExperimentationOptOutParameter == true + } + assert !cases.isEmpty() + return cases + } + + private static Map expectedFeatureMetadata(final Map assignment, final Map prefixes) { + final result = [:] + assignment.features?.each { feature -> + feature.destinations.each { destination -> + result[prefixes[destination] + feature.key] = feature.value + } + } + return result + } + private static URL fetchResource(final String name) { return Thread.currentThread().getContextClassLoader().getResource(name) } diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index 0c467473aa8..5e2430d9ad3 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit 0c467473aa8dc1f1cbfc0ef9a60d5c1c7a8826e6 +Subproject commit 5e2430d9ad36644e2cecb6a32ddb8d05d7c98ac6 diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index 4885e33b2a1..ed9a49fc2bc 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -106,6 +106,40 @@ automatic flag-related span enrichment. It does not change flag values, evaluati application defaults, experiment assignment, or exposure logging. Application hooks and unrelated tracing remain active. Previously recorded or queued data is not removed. +## Per-subject experimentation opt-out + +Enable experimentation opt-out once. Pass the subject's decision with each evaluation: + +```java +OpenFeatureAPI api = OpenFeatureAPI.getInstance(); +api.setProviderAndWait(new Provider(new Provider.Options() + .enableExperimentationOptOutParameter(true))); +Client client = api.getClient(); + +boolean enabled = client.getBooleanValue("my-feature", false, + new ImmutableContext(user.getId(), Collections.singletonMap( + "dd_experimentation_opt_out", new Value(user.hasDeclinedExperimentation())))); +``` + +Only boolean `true` activates exclusion. Missing, false, null, and other values preserve ordinary +evaluation. The provider ignores the attribute unless the option is enabled at construction. + +An excluded subject skips entire experimental allocations, including control, treatment, and +holdout groups. Evaluation continues with ordinary allocations. A completed winner rollout remains +available. Exclusion does not force the flag value to `false` or select experimental control. +Skipped allocations produce no exposures. Evaluation telemetry remains enabled unless its separate +opt-out control applies. Applications can enable both controls independently. + +The configuration must declare `experimentation-opt-out-v1` in `evaluationCapabilities`. +If this compiler guarantee is absent, an opted-out evaluation returns an error and the application's +default value. Use compatible Datadog agent and provider versions; an older agent cannot preserve +this guarantee. Ordinary evaluations keep their existing behavior. + +The evaluator tests use the fixture submodule at +`dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data`. Its +`experimentation-opt-out/v1/manifest.json` defines all 136 evaluation cases and eligible-subject +parity checks. The Java runner tests JSON, which is the Java agent's configuration format. + ## Evaluation metrics When `DD_METRICS_OTEL_ENABLED=true` and the OpenTelemetry API is on the classpath, the provider diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 2afec791851..c650d6b3f6c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -29,6 +29,7 @@ import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.Clock; import java.time.Instant; import java.util.AbstractMap; import java.util.ArrayList; @@ -213,11 +214,17 @@ static boolean exposureSerialIdSupported(final Class eventClass) { private static final boolean SPAN_ENRICHMENT_ENABLED = SpanEnrichmentGate.isEnabled(); private final Runnable configCallback; + private final Clock clock; private final AtomicReference configuration = new AtomicReference<>(); private final CountDownLatch initializationLatch = new CountDownLatch(1); public DDEvaluator(final Runnable configCallback) { + this(configCallback, Clock.systemUTC()); + } + + DDEvaluator(final Runnable configCallback, final Clock clock) { this.configCallback = configCallback; + this.clock = clock; } @Override @@ -255,6 +262,16 @@ public ProviderEvaluation evaluate( final String key, final T defaultValue, final EvaluationContext context) { + return evaluate(target, key, defaultValue, context, false); + } + + @Override + public ProviderEvaluation evaluate( + final Class target, + final String key, + final T defaultValue, + final EvaluationContext context, + final boolean experimentationOptOut) { // Snapshot the config once and thread observeFullEvaluationData through every // ProviderEvaluation returned, so the hook's consent decision is pinned to this evaluation's // config and cannot drift on a concurrent Remote Config swap. @@ -272,6 +289,14 @@ public ProviderEvaluation evaluate( return error(defaultValue, ErrorCode.INVALID_CONTEXT, null, observeFullEvaluationData); } + if (experimentationOptOut && !supportsExperimentationOptOut(config)) { + return error( + defaultValue, + ErrorCode.GENERAL, + "Configuration does not support experimentation opt-out", + observeFullEvaluationData); + } + final Flag flag = config.flags.get(key); if (flag == null) { if (config.invalidFlags != null && config.invalidFlags.containsKey(key)) { @@ -300,11 +325,28 @@ public ProviderEvaluation evaluate( observeFullEvaluationData); } - final Instant now = Instant.now(); + if (experimentationOptOut) { + // Validate the whole flag before resolving an earlier ordinary allocation. Missing + // classification must not silently allow participation with a malformed configuration. + for (final Allocation allocation : flag.allocations) { + if (allocation == null || allocation.doLog == null) { + return error( + defaultValue, + ErrorCode.PARSE_ERROR, + "Missing experimentation classification for flag " + key, + observeFullEvaluationData); + } + } + } + + final Instant now = clock.instant(); final long evalTimestampMs = now.toEpochMilli(); final String targetingKey = context.getTargetingKey(); for (final Allocation allocation : flag.allocations) { + if (experimentationOptOut && Boolean.TRUE.equals(allocation.doLog)) { + continue; + } if (!isAllocationActive(allocation, now)) { continue; } @@ -375,6 +417,16 @@ public ProviderEvaluation evaluate( } } + private static boolean supportsExperimentationOptOut(final ServerConfiguration config) { + try { + return config.evaluationCapabilities != null + && config.evaluationCapabilities.contains("experimentation-opt-out-v1"); + } catch (final NoSuchFieldError e) { + // An older agent cannot preserve the compiler capability. Do not admit an opted-out subject. + return false; + } + } + private static ImmutableMetadata consentMetadata(final boolean observeFullEvaluationData) { return ImmutableMetadata.builder() .addBoolean(METADATA_OBSERVE_FULL_EVALUATION_DATA, observeFullEvaluationData) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Evaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Evaluator.java index f4c9cacffdb..0cae9cc31f7 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Evaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Evaluator.java @@ -1,7 +1,9 @@ package datadog.trace.api.openfeature; +import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.Reason; import java.util.concurrent.TimeUnit; interface Evaluator { @@ -14,4 +16,21 @@ interface Evaluator { ProviderEvaluation evaluate( Class target, String key, T defaultValue, EvaluationContext context); + + default ProviderEvaluation evaluate( + Class target, + String key, + T defaultValue, + EvaluationContext context, + boolean experimentationOptOut) { + if (experimentationOptOut) { + return ProviderEvaluation.builder() + .value(defaultValue) + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.GENERAL) + .errorMessage("The evaluator does not support experimentation opt-out") + .build(); + } + return evaluate(target, key, defaultValue, context); + } } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index d17ac0578c1..8bb4946af3a 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -35,6 +35,7 @@ public class Provider extends EventProvider implements Metadata { private static final Options DEFAULT_OPTIONS = new Options().initTimeout(30, SECONDS); private volatile Evaluator evaluator; private final Options options; + private final boolean experimentationOptOutParameterEnabled; private final AtomicReference initializationState = new AtomicReference<>(InitializationState.NOT_STARTED); private final FlagEvalMetrics flagEvalMetrics; @@ -66,6 +67,7 @@ public Provider(final Options options) { final Evaluator evaluator, final Boolean spanEnrichmentEnabledOverride) { this.options = options; + this.experimentationOptOutParameterEnabled = options.experimentationOptOutParameterEnabled; this.evaluator = evaluator; FlagEvalMetrics metrics = null; FlagEvalMetricsHook hook = null; @@ -278,31 +280,45 @@ public String getName() { @Override public ProviderEvaluation getBooleanEvaluation( final String key, final Boolean defaultValue, final EvaluationContext ctx) { - return evaluator.evaluate(Boolean.class, key, defaultValue, ctx); + return evaluate(Boolean.class, key, defaultValue, ctx); } @Override public ProviderEvaluation getStringEvaluation( final String key, final String defaultValue, final EvaluationContext ctx) { - return evaluator.evaluate(String.class, key, defaultValue, ctx); + return evaluate(String.class, key, defaultValue, ctx); } @Override public ProviderEvaluation getIntegerEvaluation( final String key, final Integer defaultValue, final EvaluationContext ctx) { - return evaluator.evaluate(Integer.class, key, defaultValue, ctx); + return evaluate(Integer.class, key, defaultValue, ctx); } @Override public ProviderEvaluation getDoubleEvaluation( final String key, final Double defaultValue, final EvaluationContext ctx) { - return evaluator.evaluate(Double.class, key, defaultValue, ctx); + return evaluate(Double.class, key, defaultValue, ctx); } @Override public ProviderEvaluation getObjectEvaluation( final String key, final Value defaultValue, final EvaluationContext ctx) { - return evaluator.evaluate(Value.class, key, defaultValue, ctx); + return evaluate(Value.class, key, defaultValue, ctx); + } + + private ProviderEvaluation evaluate( + final Class target, + final String key, + final T defaultValue, + final EvaluationContext context) { + if (experimentationOptOutParameterEnabled && context != null) { + final Value value = context.getValue("dd_experimentation_opt_out"); + if (value != null && value.isBoolean() && Boolean.TRUE.equals(value.asBoolean())) { + return evaluator.evaluate(target, key, defaultValue, context, true); + } + } + return evaluator.evaluate(target, key, defaultValue, context); } @SuppressForbidden // Class#forName(String) used to lazy-load the evaluator implementation @@ -323,6 +339,25 @@ public static class Options { private long timeout = 30; private TimeUnit unit = SECONDS; private Boolean evaluationTelemetryOptOutParameterEnabled; + private boolean experimentationOptOutParameterEnabled; + + /** + * Enables experimentation opt-out through the {@code dd_experimentation_opt_out} context + * attribute. Disabled by default. + * + *

Only boolean {@code true} excludes the subject from experimental allocations. Evaluation + * continues with ordinary allocations, including completed winner rollouts. Skipped allocations + * do not produce exposures. Evaluation telemetry remains independent. Incompatible + * configuration produces an error and the application's default value instead of an + * experimental assignment. + * + * @param enabled whether to honor the attribute; configure before constructing the provider + * @return these options + */ + public Options enableExperimentationOptOutParameter(final boolean enabled) { + this.experimentationOptOutParameterEnabled = enabled; + return this; + } /** * Enables per-evaluation telemetry opt-out through the {@code dd_evaluation_telemetry_opt_out} diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/EvaluationTelemetryOptOutTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/EvaluationTelemetryOptOutTest.java index e82d9b7ea23..2cd1fa814f7 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/EvaluationTelemetryOptOutTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/EvaluationTelemetryOptOutTest.java @@ -1,5 +1,6 @@ package datadog.trace.api.openfeature; +import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -555,6 +556,114 @@ void missingConfigurationKeepsTheDefaultWithoutEmittingTelemetry() throws Except assertEquals(1, customerEvaluations.size()); } + @ParameterizedTest + @MethodSource("independentOptOutControls") + void experimentationAndEvaluationTelemetryRemainIndependent( + final boolean experimentationOptOut, final boolean telemetryOptOut) throws Exception { + startWithOrdinaryFallback(); + final MutableContext context = + new MutableContext("subject-1") + .add("dd_experimentation_opt_out", experimentationOptOut) + .add(PARAMETER, telemetryOptOut); + + final FlagEvaluationDetails result = client.getIntegerDetails("flag", 0, context); + + assertNull(result.getErrorCode()); + assertEquals(experimentationOptOut ? 2 : 1, result.getValue()); + assertEquals( + experimentationOptOut ? "ordinary" : "experiment", + result.getFlagMetadata().getString("allocationKey")); + assertEquals(experimentationOptOut ? 0 : 1, exposures.size()); + // The regular test JVM has the evaluator's span-enrichment gate off. Ordinary allocations + // then carry no serial metadata, independently of either opt-out setting. + assertTelemetryCounts( + telemetryOptOut ? 0 : 1, telemetryOptOut || experimentationOptOut ? 0 : 1); + assertEquals(1, customerEvaluations.size()); + } + + static Stream independentOptOutControls() { + return Stream.of( + Arguments.of(false, false), + Arguments.of(false, true), + Arguments.of(true, false), + Arguments.of(true, true)); + } + + @Test + void excludedEvaluationDoesNotReserveAnExposureDeduplicationEntry() throws Exception { + startWithOrdinaryFallback(); + final MutableContext context = + new MutableContext("subject-1").add("dd_experimentation_opt_out", true); + assertEquals(2, client.getIntegerValue("flag", 0, context)); + assertEquals(0, exposures.size()); + + context.add("dd_experimentation_opt_out", false); + assertEquals(1, client.getIntegerValue("flag", 0, context)); + assertEquals(1, exposures.size()); + assertEquals(1, client.getIntegerValue("flag", 0, context)); + assertEquals(1, exposures.size()); + assertTelemetryCounts(3, 2); + } + + @Test + void applicationBeforeHookCanSetExperimentationPreference() throws Exception { + startWithOrdinaryFallback(); + api.addHooks( + new Hook() { + @Override + public Optional before( + final HookContext context, final Map hints) { + return Optional.of(new MutableContext().add("dd_experimentation_opt_out", true)); + } + }); + assertEquals(2, client.getIntegerValue("flag", 0, new MutableContext("subject-1"))); + assertEquals(0, exposures.size()); + assertTelemetryCounts(1, 0); + } + + private void startWithOrdinaryFallback() throws Exception { + final Map variants = new java.util.HashMap<>(); + variants.put("on", new Variant("on", 1)); + variants.put("ordinary", new Variant("ordinary", 2)); + final Flag flag = + new Flag( + "flag", + true, + ValueType.INTEGER, + variants, + asList( + new Allocation( + "experiment", + null, + null, + null, + singletonList(new Split(emptyList(), "on", emptyMap(), 7)), + true), + new Allocation( + "ordinary", + null, + null, + null, + singletonList(new Split(emptyList(), "ordinary", emptyMap(), 8)), + false))); + FeatureFlaggingGateway.dispatch( + new ServerConfiguration( + "", + "", + false, + null, + singletonMap("flag", flag), + singletonList("experimentation-opt-out-v1"))); + api.setProviderAndWait( + new Provider( + new Provider.Options() + .enableExperimentationOptOutParameter(true) + .enableEvaluationTelemetryOptOutParameter(true), + new DDEvaluator(() -> {}), + true)); + client = api.getClient(); + } + private void start() throws Exception { start(new Provider.Options().enableEvaluationTelemetryOptOutParameter(true)); } @@ -591,9 +700,13 @@ private static EvaluationContext context(final String subject, final Value optOu } private void assertTelemetryCounts(final int expected) { + assertTelemetryCounts(expected, expected); + } + + private void assertTelemetryCounts(final int expected, final int expectedSpans) { assertAll( () -> assertEquals(expected, events.size(), "evaluation EVP enqueue count"), () -> assertEquals(expected, metrics.get(), "OTel evaluation count"), - () -> assertEquals(expected, spans.size(), "automatic span enrichment count")); + () -> assertEquals(expectedSpans, spans.size(), "automatic span enrichment count")); } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ExperimentationOptOutForkedTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ExperimentationOptOutForkedTest.java new file mode 100644 index 00000000000..305f83bcf27 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ExperimentationOptOutForkedTest.java @@ -0,0 +1,513 @@ +package datadog.trace.api.openfeature; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonWriter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import datadog.trace.api.featureflag.ufc.v1.Feature; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.Value; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +/** Runs the shared contract through the provider API and the real Java evaluator. */ +class ExperimentationOptOutForkedTest { + static { + // Preserve ordinary allocation serial IDs in metadata so all identities can be asserted. + System.setProperty("dd.experimental.flagging.provider.span.enrichment.enabled", "true"); + } + + private static final Moshi MOSHI = new Moshi.Builder().add(Date.class, new DateAdapter()).build(); + private static final JsonAdapter CONFIG_ADAPTER = + MOSHI.adapter(ServerConfiguration.class); + private static final JsonAdapter> CASE_ADAPTER = + MOSHI.adapter(Types.newParameterizedType(List.class, Case.class)); + + @ParameterizedTest(name = "{0}") + @MethodSource("cases") + void sharedContract(final Case testCase) throws Exception { + final DDEvaluator evaluator = evaluator(testCase); + final Provider provider = new Provider(options(testCase), evaluator); + try { + assertCase(testCase, evaluate(provider, testCase)); + } finally { + provider.shutdown(); + } + } + + @Test + void executesEveryManifestCase() throws Exception { + final List cases = cases(); + final Manifest manifest = manifest(); + assertEquals(136, manifest.expectedCounts.get("evaluations").intValue()); + assertEquals(manifest.expectedCounts.get("evaluations").intValue(), cases.size()); + assertEquals(cases.size(), cases.stream().map(item -> item.id).distinct().count()); + for (final Case testCase : cases) { + final Value value = context(testCase).getValue("dd_experimentation_opt_out"); + final boolean effective = + Boolean.TRUE.equals(testCase.providerOptions.get("enableExperimentationOptOutParameter")) + && value != null + && value.isBoolean() + && Boolean.TRUE.equals(value.asBoolean()); + assertEquals(testCase.expected.effectiveExperimentationOptOut, effective, testCase.id); + } + } + + @Test + void eligibleSubjectsKeepExactAssignmentIdentityAndReason() throws Exception { + final Manifest manifest = manifest(); + final Set ids = new HashSet<>(manifest.invariants.eligibleParityCaseIds); + ids.addAll(manifest.compilerGeneration.eligibleParityCaseIds); + int checked = 0; + for (final Case testCase : cases()) { + if (!ids.contains(testCase.id)) { + continue; + } + final DDEvaluator evaluator = evaluator(testCase); + final Provider baseline = new Provider(new Provider.Options(), evaluator); + final Provider candidate = new Provider(options(testCase), evaluator); + try { + final ProviderEvaluation before = evaluate(baseline, testCase); + final ProviderEvaluation after = evaluate(candidate, testCase); + assertCase(testCase, before); + assertCase(testCase, after); + assertEquals(before.getReason(), after.getReason(), testCase.id); + assertEquals( + before.getFlagMetadata().asUnmodifiableMap(), + after.getFlagMetadata().asUnmodifiableMap(), + testCase.id); + checked++; + } finally { + baseline.shutdown(); + candidate.shutdown(); + } + } + assertEquals(47, checked); + assertEquals(ids.size(), checked); + } + + @Test + void concurrentSubjectsUseIndependentOptions() throws Exception { + final Manifest manifest = manifest(); + final Map byId = + cases().stream().collect(Collectors.toMap(item -> item.id, item -> item)); + final Case eligible = byId.get(manifest.invariants.concurrentServerCaseIds.get(0)); + final Case excluded = byId.get(manifest.invariants.concurrentServerCaseIds.get(1)); + final Provider provider = new Provider(options(excluded), evaluator(excluded)); + final ExecutorService executor = Executors.newFixedThreadPool(8); + try { + final List> results = new ArrayList<>(); + for (int i = 0; i < manifest.invariants.concurrentServerRepetitions; i++) { + results.add(executor.submit(() -> assertCase(eligible, evaluate(provider, eligible)))); + results.add(executor.submit(() -> assertCase(excluded, evaluate(provider, excluded)))); + } + for (final Future result : results) { + result.get(); + } + assertEquals(128, results.size()); + } finally { + executor.shutdownNow(); + provider.shutdown(); + } + } + + @Test + void activationIsCapturedAtProviderConstruction() throws Exception { + final Case testCase = caseById("excluded-control"); + final Provider.Options options = new Provider.Options(); + final DDEvaluator evaluator = evaluator(testCase); + final Provider disabled = new Provider(options, evaluator); + options.enableExperimentationOptOutParameter(true); + final Provider enabled = new Provider(options, evaluator); + options.enableExperimentationOptOutParameter(false); + try { + assertEquals( + "experiment", evaluate(disabled, testCase).getFlagMetadata().getString("allocationKey")); + assertCase(testCase, evaluate(enabled, testCase)); + } finally { + disabled.shutdown(); + enabled.shutdown(); + } + } + + @Test + void legacyMissingDoLogKeepsItsOrdinaryBehavior() throws Exception { + final Case testCase = caseById("missing-dolog-excluded"); + final Provider provider = new Provider(new Provider.Options(), evaluator(testCase)); + try { + final ProviderEvaluation result = evaluate(provider, testCase); + assertEquals("experimental-control", result.getValue()); + assertEquals("experiment", result.getFlagMetadata().getString("allocationKey")); + assertFalse(result.getFlagMetadata().getBoolean(DDEvaluator.METADATA_DO_LOG)); + } finally { + provider.shutdown(); + } + } + + @Test + void olderAgentWithoutCapabilityFieldReturnsDefaultInsteadOfLinkageFailure() throws Exception { + final ClassLoader loader = new LegacyConfigurationClassLoader(); + final Class configurationType = loader.loadClass(ServerConfiguration.class.getName()); + assertThrows( + NoSuchFieldException.class, () -> configurationType.getField("evaluationCapabilities")); + final Object configuration = + configurationType + .getConstructor( + String.class, + String.class, + Boolean.class, + datadog.trace.api.featureflag.ufc.v1.Environment.class, + Map.class) + .newInstance("", "SERVER", false, null, new HashMap<>()); + final Class evaluatorType = loader.loadClass(DDEvaluator.class.getName()); + final Constructor constructor = evaluatorType.getConstructor(Runnable.class); + constructor.setAccessible(true); + final Object evaluator = constructor.newInstance((Runnable) () -> {}); + final Method accept = evaluatorType.getMethod("accept", configurationType); + accept.setAccessible(true); + accept.invoke(evaluator, configuration); + final Method evaluate = + evaluatorType.getMethod( + "evaluate", + Class.class, + String.class, + Object.class, + dev.openfeature.sdk.EvaluationContext.class, + boolean.class); + evaluate.setAccessible(true); + final ProviderEvaluation result = + (ProviderEvaluation) + evaluate.invoke( + evaluator, + String.class, + "flag", + "application-default", + new MutableContext("subject"), + true); + assertEquals("application-default", result.getValue()); + assertEquals(ErrorCode.GENERAL, result.getErrorCode()); + assertEquals("ERROR", result.getReason()); + } + + private static void assertCase(final Case testCase, final ProviderEvaluation result) { + assertEquals(testCase.expected.value, result.getValue(), testCase.id); + if (testCase.expected.assignment == null) { + assertNull(result.getFlagMetadata().getString("allocationKey"), testCase.id); + if ("no-assignment".equals(testCase.expected.outcome)) { + assertNull(result.getErrorCode(), testCase.id); + assertEquals("DEFAULT", result.getReason(), testCase.id); + } else { + assertEquals("ERROR", result.getReason(), testCase.id); + assertEquals( + "invalid-configuration".equals(testCase.expected.outcome) + ? ErrorCode.PARSE_ERROR + : ErrorCode.GENERAL, + result.getErrorCode(), + testCase.id); + } + return; + } + final Assignment expected = testCase.expected.assignment; + assertNull(result.getErrorCode(), testCase.id); + assertEquals(expected.variationKey, result.getVariant(), testCase.id); + assertEquals( + expected.allocationKey, result.getFlagMetadata().getString("allocationKey"), testCase.id); + assertEquals( + expected.doLog, + result.getFlagMetadata().getBoolean(DDEvaluator.METADATA_DO_LOG), + testCase.id); + assertEquals( + expected.serialId, + result.getFlagMetadata().getInteger(DDEvaluator.METADATA_SPLIT_SERIAL_ID), + testCase.id); + final Map expectedFeatures = new HashMap<>(); + final List features = expected.features == null ? emptyList() : expected.features; + for (final Feature feature : features) { + for (final String destination : feature.destinations) { + final String prefix; + switch (destination) { + case "HOOK": + prefix = DDEvaluator.METADATA_FEATURE_PREFIX; + break; + case "EXPOSURE": + prefix = DDEvaluator.METADATA_EXPOSURE_FEATURE_PREFIX; + break; + case "EVALUATION": + prefix = DDEvaluator.METADATA_EVALUATION_FEATURE_PREFIX; + break; + default: + throw new AssertionError("Unknown fixture destination " + destination); + } + expectedFeatures.put(prefix + feature.key, feature.value); + } + } + final Map actualFeatures = new HashMap<>(); + result + .getFlagMetadata() + .asUnmodifiableMap() + .forEach( + (key, value) -> { + if (key.startsWith(DDEvaluator.METADATA_FEATURE_PREFIX) + || key.startsWith(DDEvaluator.METADATA_EXPOSURE_FEATURE_PREFIX) + || key.startsWith(DDEvaluator.METADATA_EVALUATION_FEATURE_PREFIX)) { + actualFeatures.put(key, value); + } + }); + assertEquals(expectedFeatures, actualFeatures, testCase.id); + } + + private static ProviderEvaluation evaluate(final Provider provider, final Case testCase) { + final MutableContext context = context(testCase); + switch (testCase.evaluation.variationType) { + case "STRING": + return provider.getStringEvaluation( + testCase.evaluation.flag, (String) testCase.evaluation.defaultValue, context); + case "BOOLEAN": + return provider.getBooleanEvaluation( + testCase.evaluation.flag, (Boolean) testCase.evaluation.defaultValue, context); + default: + throw new AssertionError("Unsupported fixture type " + testCase.evaluation.variationType); + } + } + + private static MutableContext context(final Case testCase) { + return new MutableContext( + testCase.context.targetingKey, + Value.objectToValue(testCase.context.attributes).asStructure().asMap()); + } + + private static Provider.Options options(final Case testCase) { + return new Provider.Options() + .enableExperimentationOptOutParameter( + Boolean.TRUE.equals( + testCase.providerOptions.get("enableExperimentationOptOutParameter"))) + .enableEvaluationTelemetryOptOutParameter( + Boolean.TRUE.equals( + testCase.providerOptions.get("enableEvaluationTelemetryOptOutParameter"))); + } + + private static DDEvaluator evaluator(final Case testCase) throws IOException { + final DDEvaluator evaluator = + new DDEvaluator( + () -> {}, Clock.fixed(Instant.parse(testCase.evaluation.now), ZoneOffset.UTC)); + final Configuration configuration = + manifest().configurations.stream() + .filter(item -> item.id.equals(testCase.configuration)) + .findFirst() + .orElseThrow(AssertionError::new); + evaluator.accept(CONFIG_ADAPTER.fromJson(read(root().resolve(configuration.path)))); + return evaluator; + } + + private static Case caseById(final String id) throws IOException { + return cases().stream() + .filter(item -> item.id.equals(id)) + .findFirst() + .orElseThrow(AssertionError::new); + } + + private static List cases() throws IOException { + final List result = new ArrayList<>(); + for (final CaseFile file : manifest().evaluationCases) { + final List items = CASE_ADAPTER.fromJson(read(root().resolve(file.path))); + assertNotNull(items); + assertEquals( + file.ids, items.stream().map(item -> item.id).collect(Collectors.toList()), file.path); + result.addAll(items); + } + return result; + } + + private static Manifest manifest() throws IOException { + return MOSHI.adapter(Manifest.class).fromJson(read(root().resolve("manifest.json"))); + } + + private static Path root() { + Path directory = Paths.get("").toAbsolutePath(); + while (directory != null) { + final Path root = + directory.resolve( + "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/experimentation-opt-out/v1"); + if (Files.exists(root.resolve("manifest.json"))) { + return root; + } + directory = directory.getParent(); + } + throw new AssertionError("Initialize the pinned FFE fixture submodule"); + } + + private static String read(final Path path) throws IOException { + return new String(Files.readAllBytes(path), UTF_8); + } + + static class Manifest { + List configurations; + List evaluationCases; + Map expectedCounts; + Invariants invariants; + CompilerGeneration compilerGeneration; + } + + static class Configuration { + String id; + String path; + } + + static class CaseFile { + String path; + List ids; + } + + static class Invariants { + List eligibleParityCaseIds; + List concurrentServerCaseIds; + int concurrentServerRepetitions; + } + + static class CompilerGeneration { + List eligibleParityCaseIds; + } + + static class Case { + String id; + String configuration; + Map providerOptions; + Context context; + Evaluation evaluation; + Expected expected; + + @Override + public String toString() { + return id; + } + } + + static class Context { + String targetingKey; + Map attributes; + } + + static class Evaluation { + String flag; + String variationType; + Object defaultValue; + String now; + } + + static class Expected { + boolean effectiveExperimentationOptOut; + String outcome; + Object value; + Assignment assignment; + } + + static class Assignment { + String allocationKey; + String variationKey; + Boolean doLog; + Integer serialId; + List features; + } + + static class DateAdapter extends JsonAdapter { + @Override + public Date fromJson(final JsonReader reader) throws IOException { + if (reader.peek() == JsonReader.Token.NULL) { + return reader.nextNull(); + } + return Date.from(Instant.parse(reader.nextString())); + } + + @Override + public void toJson(final JsonWriter writer, final Date value) throws IOException { + if (value == null) { + writer.nullValue(); + } else { + writer.value(value.toInstant().toString()); + } + } + } + + /** Loads the provider against a bootstrap configuration class that lacks the new field. */ + static class LegacyConfigurationClassLoader extends ClassLoader { + LegacyConfigurationClassLoader() { + super(ExperimentationOptOutForkedTest.class.getClassLoader()); + } + + @Override + protected synchronized Class loadClass(final String name, final boolean resolve) + throws ClassNotFoundException { + if (!name.startsWith("datadog.trace.api.openfeature.") + && !name.equals(ServerConfiguration.class.getName())) { + return super.loadClass(name, resolve); + } + Class type = findLoadedClass(name); + if (type == null) { + try (InputStream source = + getParent().getResourceAsStream(name.replace('.', '/') + ".class")) { + assertNotNull(source, name); + final byte[] bytes = source.readAllBytes(); + if (name.equals(ServerConfiguration.class.getName())) { + final byte[] field = "evaluationCapabilities".getBytes(UTF_8); + final byte[] legacy = "obsoleteCapabilities__".getBytes(UTF_8); + assertEquals(field.length, legacy.length); + int replacements = 0; + for (int i = 0; i <= bytes.length - field.length; i++) { + boolean matches = true; + for (int j = 0; j < field.length; j++) { + matches &= bytes[i + j] == field[j]; + } + if (matches) { + System.arraycopy(legacy, 0, bytes, i, legacy.length); + replacements++; + } + } + assertTrue(replacements > 0); + } + type = defineClass(name, bytes, 0, bytes.length); + } catch (final IOException e) { + throw new ClassNotFoundException(name, e); + } + } + if (resolve) { + resolveClass(type); + } + return type; + } + } +} diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/ServerConfiguration.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/ServerConfiguration.java index 60f39fc666f..b6378be62a8 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/ServerConfiguration.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/ServerConfiguration.java @@ -1,6 +1,7 @@ package datadog.trace.api.featureflag.ufc.v1; import java.util.Collections; +import java.util.List; import java.util.Map; public class ServerConfiguration { @@ -15,6 +16,9 @@ public class ServerConfiguration { public final Environment environment; public final Map flags; + /** Compiler guarantees required by optional evaluation behavior. */ + public final List evaluationCapabilities; + // Flags that could not be parsed or validated. The key is the flag key; the value is the error // type (e.g. "invalid_semver_comparand"). Set during configuration preprocessing, not from JSON. public transient Map invalidFlags = Collections.emptyMap(); @@ -25,10 +29,21 @@ public ServerConfiguration( final Boolean observeFullEvaluationData, final Environment environment, final Map flags) { + this(createdAt, format, observeFullEvaluationData, environment, flags, null); + } + + public ServerConfiguration( + final String createdAt, + final String format, + final Boolean observeFullEvaluationData, + final Environment environment, + final Map flags, + final List evaluationCapabilities) { this.createdAt = createdAt; this.format = format; this.observeFullEvaluationData = observeFullEvaluationData; this.environment = environment; this.flags = flags; + this.evaluationCapabilities = evaluationCapabilities; } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java index fae2631ce2f..f884f15dc92 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java @@ -24,6 +24,22 @@ class JsonApiUfcResponseParserTest { + @Test + void preservesCompilerEvaluationCapabilities() throws Exception { + final String configuration = + emptyConfig() + .replaceFirst("\\{", "{\"evaluationCapabilities\":[\"experimentation-opt-out-v1\"],"); + final ServerConfiguration direct = + UniversalFlagConfigParser.INSTANCE.deserialize(configuration.getBytes(UTF_8)); + final ServerConfiguration wrapped = parse(wrap(configuration)); + assertEquals( + Collections.singletonList("experimentation-opt-out-v1"), direct.evaluationCapabilities); + assertEquals(direct.evaluationCapabilities, wrapped.evaluationCapabilities); + assertNull( + UniversalFlagConfigParser.INSTANCE.deserialize(emptyConfig().getBytes(UTF_8)) + .evaluationCapabilities); + } + @Test void parsesJsonApiMembersInAnyOrder() throws Exception { final ServerConfiguration configuration =