From 92ed06e0bc5cef7e8df0643526b250bad4b422a3 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 16 Nov 2026 19:11:34 -0500 Subject: [PATCH 01/18] Benchmark the FFE scale evaluation suite Add ScaleSuiteBenchmark, which evaluates each case of the opt-in scale suite of the shared FFE fixtures. The setup checks every expected value and reason first. Pin ffe-system-test-data to the scale suite PR head. --- .../openfeature/ScaleSuiteBenchmark.java | 251 ++++++++++++++++++ .../src/test/resources/ffe-system-test-data | 2 +- 2 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java new file mode 100644 index 00000000000..a6e7329ce4c --- /dev/null +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -0,0 +1,251 @@ +package com.datadog.openfeature; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.datadog.openfeature.internal.bridge.v1.AgentBridge; +import com.datadog.openfeature.internal.config.Settings; +import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.Value; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Evaluation cost on the opt-in scale suite of the shared FFE fixtures ({@code + * evaluation-suites/scale}): 10,000-entry lists, regex conditions on 2 KB inputs, deep rules, a + * 200-attribute context and JSON values with hundreds of nodes. + * + *

Each operation evaluates the next case of the selected case file through the provider. The + * setup first evaluates every case once and fails if a value or reason differs from the expected + * one, so the benchmark never measures a wrong result. + * + *

Run: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteBenchmark}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 2, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Fork(value = 1) +public class ScaleSuiteBenchmark { + private static final String SUITE_PATH = + "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites/scale"; + private static final JsonFactory JSON_FACTORY = new JsonFactory(); + + /** The case file of the scale suite, without the {@code .json} extension. */ + @Param({"json-values", "one-of", "regex", "rules", "wide-context"}) + public String cases; + + private Provider provider; + private ScaleCase[] scaleCases; + private int next; + + @Setup(Level.Trial) + public void setUp() throws IOException { + final Path suite = suiteRoot(); + final DDEvaluator evaluator = + new DDEvaluator( + () -> {}, + AgentBridge.NONE, + Settings.load(AgentBridge.NONE), + ExperimentationOptOutMode.DISABLED); + evaluator.accept( + UniversalFlagConfigParser.parse(Files.readAllBytes(suite.resolve("ufc-config.json")))); + provider = new Provider(new Provider.Options(), evaluator, false); + + final List loaded = new ArrayList<>(); + for (final Object json : (List) read(suite.resolve("cases").resolve(cases + ".json"))) { + loaded.add(new ScaleCase((Map) json)); + } + scaleCases = loaded.toArray(new ScaleCase[0]); + for (final ScaleCase scaleCase : scaleCases) { + scaleCase.check(scaleCase.evaluate(provider)); + } + } + + @TearDown(Level.Trial) + public void tearDown() { + provider.shutdown(); + } + + @Benchmark + public ProviderEvaluation evaluate() { + final ScaleCase scaleCase = scaleCases[next]; + next = next + 1 == scaleCases.length ? 0 : next + 1; + return scaleCase.evaluate(provider); + } + + /** One evaluation case, with its expected value and reason. */ + private static final class ScaleCase { + private final String flag; + private final String variationType; + private final Object defaultValue; + private final Object expectedValue; + private final String expectedReason; + private final EvaluationContext context; + + ScaleCase(final Map json) { + this.flag = (String) json.get("flag"); + this.variationType = (String) json.get("variationType"); + final Map result = (Map) json.get("result"); + this.defaultValue = typed(json.get("defaultValue")); + this.expectedValue = typed(result.get("value")); + this.expectedReason = (String) result.get("reason"); + final MutableContext context = + new MutableContext(Value.objectToValue(json.get("attributes")).asStructure().asMap()); + context.setTargetingKey((String) json.get("targetingKey")); + this.context = context; + } + + ProviderEvaluation evaluate(final Provider provider) { + switch (variationType) { + case "BOOLEAN": + return provider.getBooleanEvaluation(flag, (Boolean) defaultValue, context); + case "INTEGER": + return provider.getIntegerEvaluation(flag, (Integer) defaultValue, context); + case "NUMERIC": + return provider.getDoubleEvaluation(flag, (Double) defaultValue, context); + case "STRING": + return provider.getStringEvaluation(flag, (String) defaultValue, context); + case "JSON": + return provider.getObjectEvaluation(flag, (Value) defaultValue, context); + default: + throw new IllegalArgumentException("Unsupported variationType: " + variationType); + } + } + + void check(final ProviderEvaluation details) { + if (!Objects.equals(expectedValue, details.getValue()) + || !Objects.equals(expectedReason, details.getReason())) { + throw new IllegalStateException( + "Unexpected result for " + + flag + + ": expected " + + expectedReason + + " " + + expectedValue + + ", got " + + details.getReason() + + " " + + details.getValue()); + } + } + + /** Converts a fixture value to the type that the provider returns for this variation type. */ + private Object typed(final Object value) { + if (value == null) { + return null; + } + switch (variationType) { + case "INTEGER": + return ((Number) value).intValue(); + case "NUMERIC": + return ((Number) value).doubleValue(); + case "JSON": + return Value.objectToValue(exactIntegers(value)); + default: + return value; + } + } + } + + /** Reads integral numbers as Integer when they fit, as the UFC parser does for JSON variants. */ + private static Object exactIntegers(final Object value) { + if (value instanceof Long) { + final long number = (Long) value; + return number == (int) number ? (Object) (int) number : value; + } + if (value instanceof Map) { + final Map map = new LinkedHashMap<>(); + ((Map) value).forEach((k, v) -> map.put((String) k, exactIntegers(v))); + return map; + } + if (value instanceof List) { + final List list = new ArrayList<>(); + ((List) value).forEach(v -> list.add(exactIntegers(v))); + return list; + } + return value; + } + + /** Reads a JSON file: integral numbers as Long, other numbers as Double. */ + private static Object read(final Path file) throws IOException { + try (JsonParser parser = JSON_FACTORY.createParser(Files.readAllBytes(file))) { + parser.nextToken(); + return readValue(parser); + } + } + + private static Object readValue(final JsonParser parser) throws IOException { + final JsonToken token = parser.currentToken(); + switch (token) { + case START_OBJECT: + final Map object = new LinkedHashMap<>(); + while (parser.nextToken() == JsonToken.FIELD_NAME) { + final String name = parser.currentName(); + parser.nextToken(); + object.put(name, readValue(parser)); + } + return object; + case START_ARRAY: + final List array = new ArrayList<>(); + while (parser.nextToken() != JsonToken.END_ARRAY) { + array.add(readValue(parser)); + } + return array; + case VALUE_STRING: + return parser.getText(); + case VALUE_NUMBER_INT: + return parser.getLongValue(); + case VALUE_NUMBER_FLOAT: + return parser.getDoubleValue(); + case VALUE_TRUE: + return Boolean.TRUE; + case VALUE_FALSE: + return Boolean.FALSE; + case VALUE_NULL: + return null; + default: + throw new IOException("Unexpected JSON token: " + token); + } + } + + private static Path suiteRoot() { + Path directory = Paths.get("").toAbsolutePath(); + while (directory != null) { + final Path candidate = directory.resolve(SUITE_PATH); + if (Files.exists(candidate.resolve("ufc-config.json"))) { + return candidate; + } + directory = directory.getParent(); + } + throw new IllegalStateException( + "Unable to find the FFE scale suite: initialize the ffe-system-test-data submodule"); + } +} diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index 6df74e605f7..0081d8fa542 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit 6df74e605f7c1eed8b397dc9aa4c002691475939 +Subproject commit 0081d8fa5421cf817491b3f3943a8b8167b593ee From c18784af78bb50a09daf047161ffb556af77969b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 17 Nov 2026 00:54:05 +0000 Subject: [PATCH 02/18] Benchmark every scale dimension on master, and the configuration load The scale benchmark sat on the JSON value change and checked JSON integers by type, so it could not set a baseline on master. It now compares JSON numbers by value, covers every case file of the scale and scale-json suites, can isolate one case with the index parameter, and prints each case's index. A load benchmark measures parsing each suite's configuration and prints the heap it retains, so changes that move work to load time show their cost. The submodule points to the scale-suite fixture branch, which adds the N x M dimensions and JSON values up to 10 MB. Co-Authored-By: Claude Opus 5.5 --- .../openfeature/ScaleSuiteBenchmark.java | 205 ++++++++++-------- .../openfeature/ScaleSuiteLoadBenchmark.java | 79 +++++++ .../com/datadog/openfeature/ScaleSuites.java | 95 ++++++++ .../src/test/resources/ffe-system-test-data | 2 +- 4 files changed, 289 insertions(+), 92 deletions(-) create mode 100644 dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java create mode 100644 dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuites.java diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index a6e7329ce4c..5efa3df3752 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -6,19 +6,12 @@ import com.datadog.openfeature.internal.bridge.v1.AgentBridge; import com.datadog.openfeature.internal.config.Settings; import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.ProviderEvaluation; import dev.openfeature.sdk.Value; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -37,15 +30,23 @@ import org.openjdk.jmh.annotations.Warmup; /** - * Evaluation cost on the opt-in scale suite of the shared FFE fixtures ({@code - * evaluation-suites/scale}): 10,000-entry lists, regex conditions on 2 KB inputs, deep rules, a - * 200-attribute context and JSON values with hundreds of nodes. + * Evaluation cost on the opt-in scale suites of the shared FFE fixtures ({@code + * evaluation-suites/scale} and {@code evaluation-suites/scale-json}): large lists, regex, rules, + * splits and shards, time windows, many variations and flags, and JSON values up to 10 MB. The + * dimensions are listed in {@code evaluation-suites/scale/AGENTS.md} of the fixtures. * - *

Each operation evaluates the next case of the selected case file through the provider. The - * setup first evaluates every case once and fails if a value or reason differs from the expected - * one, so the benchmark never measures a wrong result. + *

Each operation evaluates a case of the selected case file through the provider. With {@code + * index=all}, the operations go through the cases of the file in turn, so a row is the mean of its + * best case, worst case and miss. Select one case to measure one shape: the setup prints the index + * of each case. * - *

Run: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteBenchmark}. + *

The setup first evaluates every case once and fails if a value or reason differs from the + * expected one, so the benchmark never measures a wrong result. JSON values are compared by numeric + * value: a JSON integer can come back as an integer or as a double. + * + *

Run all case files: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteBenchmark + * -Pjmh.profilers=gc}. Run one case: {@code java -jar dd-openfeature/build/libs/*-jmh.jar + * ScaleSuiteBenchmark -p cases=scale/splits -p index=0 -prof gc}. */ @State(Scope.Benchmark) @Warmup(iterations = 3, time = 2, timeUnit = SECONDS) @@ -54,39 +55,62 @@ @OutputTimeUnit(NANOSECONDS) @Fork(value = 1) public class ScaleSuiteBenchmark { - private static final String SUITE_PATH = - "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites/scale"; - private static final JsonFactory JSON_FACTORY = new JsonFactory(); - /** The case file of the scale suite, without the {@code .json} extension. */ - @Param({"json-values", "one-of", "regex", "rules", "wide-context"}) + /** A case file, as {@code /} without the {@code .json} extension. */ + @Param({ + "scale/flags", + "scale/numeric", + "scale/one-of", + "scale/one-of-many", + "scale/regex", + "scale/regex-many", + "scale/rules", + "scale/rules-sweep", + "scale/semver", + "scale/splits", + "scale/time-windows", + "scale/variations", + "scale/wide-context", + "scale-json/json-shapes", + "scale-json/json-sizes", + "scale-json/json-values" + }) public String cases; + /** The index of one case in the file, or {@code all} to go through every case in turn. */ + @Param({"all"}) + public String index; + private Provider provider; private ScaleCase[] scaleCases; private int next; @Setup(Level.Trial) public void setUp() throws IOException { - final Path suite = suiteRoot(); + final int slash = cases.indexOf('/'); + final ScaleSuites.Suite suite = ScaleSuites.load(cases.substring(0, slash)); final DDEvaluator evaluator = new DDEvaluator( () -> {}, AgentBridge.NONE, Settings.load(AgentBridge.NONE), ExperimentationOptOutMode.DISABLED); - evaluator.accept( - UniversalFlagConfigParser.parse(Files.readAllBytes(suite.resolve("ufc-config.json")))); + evaluator.accept(UniversalFlagConfigParser.parse(suite.config())); provider = new Provider(new Provider.Options(), evaluator, false); final List loaded = new ArrayList<>(); - for (final Object json : (List) read(suite.resolve("cases").resolve(cases + ".json"))) { + for (final Object json : suite.cases(cases.substring(slash + 1))) { loaded.add(new ScaleCase((Map) json)); } - scaleCases = loaded.toArray(new ScaleCase[0]); - for (final ScaleCase scaleCase : scaleCases) { + for (int i = 0; i < loaded.size(); i++) { + final ScaleCase scaleCase = loaded.get(i); scaleCase.check(scaleCase.evaluate(provider)); + System.out.println("[" + cases + "] case " + i + ": " + scaleCase.describe()); } + scaleCases = + "all".equals(index) + ? loaded.toArray(new ScaleCase[0]) + : new ScaleCase[] {loaded.get(Integer.parseInt(index))}; } @TearDown(Level.Trial) @@ -108,6 +132,7 @@ private static final class ScaleCase { private final Object defaultValue; private final Object expectedValue; private final String expectedReason; + private final String targetingKey; private final EvaluationContext context; ScaleCase(final Map json) { @@ -115,11 +140,12 @@ private static final class ScaleCase { this.variationType = (String) json.get("variationType"); final Map result = (Map) json.get("result"); this.defaultValue = typed(json.get("defaultValue")); - this.expectedValue = typed(result.get("value")); + this.expectedValue = result.get("value"); this.expectedReason = (String) result.get("reason"); + this.targetingKey = (String) json.get("targetingKey"); final MutableContext context = new MutableContext(Value.objectToValue(json.get("attributes")).asStructure().asMap()); - context.setTargetingKey((String) json.get("targetingKey")); + context.setTargetingKey(targetingKey); this.context = context; } @@ -141,23 +167,33 @@ ProviderEvaluation evaluate(final Provider provider) { } void check(final ProviderEvaluation details) { - if (!Objects.equals(expectedValue, details.getValue()) + final Object actual = + details.getValue() instanceof Value + ? plain((Value) details.getValue()) + : details.getValue(); + if (!sameValue(expectedValue, actual) || !Objects.equals(expectedReason, details.getReason())) { throw new IllegalStateException( "Unexpected result for " + flag - + ": expected " + + " (" + + targetingKey + + "): expected " + expectedReason + " " - + expectedValue + + abbreviate(expectedValue) + ", got " + details.getReason() + " " - + details.getValue()); + + abbreviate(actual)); } } - /** Converts a fixture value to the type that the provider returns for this variation type. */ + String describe() { + return flag + " (" + targetingKey + ") -> " + expectedReason; + } + + /** Converts a fixture value to the type that the provider takes for this variation type. */ private Object typed(final Object value) { if (value == null) { return null; @@ -168,84 +204,71 @@ private Object typed(final Object value) { case "NUMERIC": return ((Number) value).doubleValue(); case "JSON": - return Value.objectToValue(exactIntegers(value)); + return Value.objectToValue(value); default: return value; } } } - /** Reads integral numbers as Integer when they fit, as the UFC parser does for JSON variants. */ - private static Object exactIntegers(final Object value) { - if (value instanceof Long) { - final long number = (Long) value; - return number == (int) number ? (Object) (int) number : value; + /** Converts a returned value to maps, lists, strings, numbers, booleans and null. */ + private static Object plain(final Value value) { + if (value == null || value.isNull()) { + return null; } - if (value instanceof Map) { - final Map map = new LinkedHashMap<>(); - ((Map) value).forEach((k, v) -> map.put((String) k, exactIntegers(v))); + if (value.isStructure()) { + final Map map = new java.util.LinkedHashMap<>(); + value.asStructure().asMap().forEach((k, v) -> map.put(k, plain(v))); return map; } - if (value instanceof List) { + if (value.isList()) { final List list = new ArrayList<>(); - ((List) value).forEach(v -> list.add(exactIntegers(v))); + for (final Value item : value.asList()) { + list.add(plain(item)); + } return list; } - return value; + return value.asObject(); } - /** Reads a JSON file: integral numbers as Long, other numbers as Double. */ - private static Object read(final Path file) throws IOException { - try (JsonParser parser = JSON_FACTORY.createParser(Files.readAllBytes(file))) { - parser.nextToken(); - return readValue(parser); + /** Equality that compares numbers by value, so 5 and 5.0 are the same. */ + private static boolean sameValue(final Object expected, final Object actual) { + if (expected instanceof Number && actual instanceof Number) { + return new java.math.BigDecimal(expected.toString()) + .compareTo(new java.math.BigDecimal(actual.toString())) + == 0; } - } - - private static Object readValue(final JsonParser parser) throws IOException { - final JsonToken token = parser.currentToken(); - switch (token) { - case START_OBJECT: - final Map object = new LinkedHashMap<>(); - while (parser.nextToken() == JsonToken.FIELD_NAME) { - final String name = parser.currentName(); - parser.nextToken(); - object.put(name, readValue(parser)); + if (expected instanceof Map && actual instanceof Map) { + final Map e = (Map) expected; + final Map a = (Map) actual; + if (e.size() != a.size()) { + return false; + } + for (final Map.Entry entry : e.entrySet()) { + if (!a.containsKey(entry.getKey()) || !sameValue(entry.getValue(), a.get(entry.getKey()))) { + return false; } - return object; - case START_ARRAY: - final List array = new ArrayList<>(); - while (parser.nextToken() != JsonToken.END_ARRAY) { - array.add(readValue(parser)); + } + return true; + } + if (expected instanceof List && actual instanceof List) { + final List e = (List) expected; + final List a = (List) actual; + if (e.size() != a.size()) { + return false; + } + for (int i = 0; i < e.size(); i++) { + if (!sameValue(e.get(i), a.get(i))) { + return false; } - return array; - case VALUE_STRING: - return parser.getText(); - case VALUE_NUMBER_INT: - return parser.getLongValue(); - case VALUE_NUMBER_FLOAT: - return parser.getDoubleValue(); - case VALUE_TRUE: - return Boolean.TRUE; - case VALUE_FALSE: - return Boolean.FALSE; - case VALUE_NULL: - return null; - default: - throw new IOException("Unexpected JSON token: " + token); + } + return true; } + return Objects.equals(expected, actual); } - private static Path suiteRoot() { - Path directory = Paths.get("").toAbsolutePath(); - while (directory != null) { - final Path candidate = directory.resolve(SUITE_PATH); - if (Files.exists(candidate.resolve("ufc-config.json"))) { - return candidate; - } - directory = directory.getParent(); - } - throw new IllegalStateException( - "Unable to find the FFE scale suite: initialize the ffe-system-test-data submodule"); + private static String abbreviate(final Object value) { + final String text = String.valueOf(value); + return text.length() <= 200 ? text : text.substring(0, 200) + "... (" + text.length() + ")"; } } diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java new file mode 100644 index 00000000000..acebb1a4dfb --- /dev/null +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java @@ -0,0 +1,79 @@ +package com.datadog.openfeature; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.datadog.openfeature.internal.ufc.ServerConfiguration; +import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Configuration load cost of the opt-in scale suites: the time to parse each suite's {@code + * ufc-config.json}. Work that an evaluator moves to load time, such as indexes, compiled patterns + * or prebuilt JSON values, shows here. + * + *

The setup also prints the heap that a parsed configuration retains. It is the used heap after + * a full collection, with and without the parsed configuration, so it is approximate. + * + *

Run: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteLoadBenchmark + * -Pjmh.profilers=gc}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 2, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(MILLISECONDS) +@Fork(value = 1) +public class ScaleSuiteLoadBenchmark { + + @Param({"scale", "scale-json"}) + public String suite; + + private byte[] config; + + @Setup(Level.Trial) + public void setUp() throws IOException { + config = ScaleSuites.load(suite).config(); + final long before = usedHeapAfterCollection(); + ServerConfiguration retained = UniversalFlagConfigParser.parse(config); + final long after = usedHeapAfterCollection(); + System.out.println( + "[" + + suite + + "] config " + + config.length / 1024 + + " KB, " + + retained.flags.size() + + " flags, retained heap about " + + Math.max(0, after - before) / 1024 + + " KB"); + retained = null; + } + + @Benchmark + public ServerConfiguration parse() throws IOException { + return UniversalFlagConfigParser.parse(config); + } + + private static long usedHeapAfterCollection() { + final MemoryMXBean memory = ManagementFactory.getMemoryMXBean(); + for (int i = 0; i < 3; i++) { + System.gc(); + } + return memory.getHeapMemoryUsage().getUsed(); + } +} diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuites.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuites.java new file mode 100644 index 00000000000..9c3b3b5b421 --- /dev/null +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuites.java @@ -0,0 +1,95 @@ +package com.datadog.openfeature; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Reads the opt-in scale suites of the ffe-system-test-data submodule. */ +final class ScaleSuites { + private static final String SUITES_PATH = + "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites"; + private static final JsonFactory JSON_FACTORY = new JsonFactory(); + + private ScaleSuites() {} + + /** One suite directory: its configuration and its case files. */ + static final class Suite { + private final Path directory; + + private Suite(final Path directory) { + this.directory = directory; + } + + byte[] config() throws IOException { + return Files.readAllBytes(directory.resolve("ufc-config.json")); + } + + /** The cases of {@code cases/.json}. */ + List cases(final String name) throws IOException { + return (List) read(directory.resolve("cases").resolve(name + ".json")); + } + } + + static Suite load(final String name) { + Path directory = Paths.get("").toAbsolutePath(); + while (directory != null) { + final Path candidate = directory.resolve(SUITES_PATH).resolve(name); + if (Files.exists(candidate.resolve("ufc-config.json"))) { + return new Suite(candidate); + } + directory = directory.getParent(); + } + throw new IllegalStateException( + "Unable to find the FFE suite " + name + ": initialize the ffe-system-test-data submodule"); + } + + /** Reads a JSON file: integral numbers as Long, other numbers as Double. */ + private static Object read(final Path file) throws IOException { + try (JsonParser parser = JSON_FACTORY.createParser(Files.readAllBytes(file))) { + parser.nextToken(); + return readValue(parser); + } + } + + private static Object readValue(final JsonParser parser) throws IOException { + final JsonToken token = parser.currentToken(); + switch (token) { + case START_OBJECT: + final Map object = new LinkedHashMap<>(); + while (parser.nextToken() == JsonToken.FIELD_NAME) { + final String name = parser.currentName(); + parser.nextToken(); + object.put(name, readValue(parser)); + } + return object; + case START_ARRAY: + final List array = new ArrayList<>(); + while (parser.nextToken() != JsonToken.END_ARRAY) { + array.add(readValue(parser)); + } + return array; + case VALUE_STRING: + return parser.getText(); + case VALUE_NUMBER_INT: + return parser.getLongValue(); + case VALUE_NUMBER_FLOAT: + return parser.getDoubleValue(); + case VALUE_TRUE: + return Boolean.TRUE; + case VALUE_FALSE: + return Boolean.FALSE; + case VALUE_NULL: + return null; + default: + throw new IOException("Unexpected JSON token: " + token); + } + } +} diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index 0081d8fa542..af0188a2b4f 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit 0081d8fa5421cf817491b3f3943a8b8167b593ee +Subproject commit af0188a2b4ff08ae6f9740ca89c904b812750ba7 From a8876576426769b6211863104c435c60f67b0ebf Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 17 Nov 2026 04:00:01 +0000 Subject: [PATCH 03/18] Check the scale suites in the unit tests and harden the benchmarks The scale benchmarks failed the forbidden API check on System.out, and nothing outside a manual JMH run checked the scale cases, so a sibling change that resolves a large configuration wrongly would go unnoticed. - Add ScaleSuiteTest: evaluate every case of the scale and scale-json suites twice, and fail when the benchmark misses a case file. The cases, their check and the suite reader move to the test sources and the benchmarks reuse them; the reader uses JsonReading. - Allow System.out in the benchmark setups (@SuppressForbidden). - Check every case again after the measurement, fail on an error code, and reject expectations the check does not handle. - Rerun the jmh task when the fixtures change. - Measure retained heap after a first parse, as the smallest of three signed samples. - Describe a row as the unweighted mean over the cases of a file, say that hooks are not measured and when to run three forks. Co-Authored-By: Claude Opus 5.5 --- dd-openfeature/build.gradle.kts | 9 + .../openfeature/ScaleSuiteBenchmark.java | 207 +++--------------- .../openfeature/ScaleSuiteLoadBenchmark.java | 25 ++- .../com/datadog/openfeature/ScaleSuites.java | 95 -------- .../com/datadog/openfeature/ScaleCase.java | 168 ++++++++++++++ .../datadog/openfeature/ScaleSuiteTest.java | 75 +++++++ .../com/datadog/openfeature/ScaleSuites.java | 93 ++++++++ 7 files changed, 392 insertions(+), 280 deletions(-) delete mode 100644 dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuites.java create mode 100644 dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java create mode 100644 dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java create mode 100644 dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java diff --git a/dd-openfeature/build.gradle.kts b/dd-openfeature/build.gradle.kts index ecd5d79228c..65835411e02 100644 --- a/dd-openfeature/build.gradle.kts +++ b/dd-openfeature/build.gradle.kts @@ -160,3 +160,12 @@ tasks.withType().configureEach { ) ).withPathSensitivity(PathSensitivity.RELATIVE) } + +// The scale benchmarks read the fixtures from disk at run time, so a fixture change reruns them. +tasks.named("jmh") { + inputs.dir( + rootProject.layout.projectDirectory.dir( + "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites" + ) + ).withPathSensitivity(PathSensitivity.RELATIVE) +} diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index 5efa3df3752..b0ad49fccb9 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -3,18 +3,10 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; -import com.datadog.openfeature.internal.bridge.v1.AgentBridge; -import com.datadog.openfeature.internal.config.Settings; -import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; -import dev.openfeature.sdk.EvaluationContext; -import dev.openfeature.sdk.MutableContext; +import de.thetaphi.forbiddenapis.SuppressForbidden; import dev.openfeature.sdk.ProviderEvaluation; -import dev.openfeature.sdk.Value; import java.io.IOException; -import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Objects; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -36,17 +28,27 @@ * dimensions are listed in {@code evaluation-suites/scale/AGENTS.md} of the fixtures. * *

Each operation evaluates a case of the selected case file through the provider. With {@code - * index=all}, the operations go through the cases of the file in turn, so a row is the mean of its - * best case, worst case and miss. Select one case to measure one shape: the setup prints the index - * of each case. + * index=all}, the operations go through the cases of the file in turn, so a row is the unweighted + * mean over every case of the file, and its slowest case dominates it. Select one case to measure + * one shape: the setup prints the index of each case. * - *

The setup first evaluates every case once and fails if a value or reason differs from the - * expected one, so the benchmark never measures a wrong result. JSON values are compared by numeric - * value: a JSON integer can come back as an integer or as a double. + *

The setup evaluates every case once and the teardown again after the measurement. Both fail if + * a value or reason differs from the expected one, or if an evaluation reports an error, so a run + * that resolves a case wrongly fails instead of reporting a number. JSON values are compared by + * numeric value: a JSON integer can come back as an integer or as a double. This check guards the + * measurement and is not the conformance check: {@code CanonicalFixtureTest} checks the types. + * + *

The provider is called directly with a reused context: the OpenFeature client, its hooks and + * exposure logging are not measured. + * + *

The cases and their check are shared with {@code ScaleSuiteTest}, which runs them in the unit + * tests. One fork is enough for allocation and for large time differences. For a time difference + * under about 15%, run three forks: {@code -Pjmh.forks=3} or {@code -f 3}. * *

Run all case files: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteBenchmark - * -Pjmh.profilers=gc}. Run one case: {@code java -jar dd-openfeature/build/libs/*-jmh.jar - * ScaleSuiteBenchmark -p cases=scale/splits -p index=0 -prof gc}. + * -Pjmh.profilers=gc}. Run one case: {@code ./gradlew :dd-openfeature:jmhJar}, then {@code java + * -jar dd-openfeature/build/libs/*-jmh.jar ScaleSuiteBenchmark -p cases=scale/splits -p index=0 + * -prof gc}. */ @State(Scope.Benchmark) @Warmup(iterations = 3, time = 2, timeUnit = SECONDS) @@ -56,7 +58,10 @@ @Fork(value = 1) public class ScaleSuiteBenchmark { - /** A case file, as {@code /} without the {@code .json} extension. */ + /** + * A case file, as {@code /} without the {@code .json} extension. {@code + * ScaleSuiteTest} fails when this list misses a case file of the fixtures. + */ @Param({ "scale/flags", "scale/numeric", @@ -86,22 +91,12 @@ public class ScaleSuiteBenchmark { private int next; @Setup(Level.Trial) + @SuppressForbidden public void setUp() throws IOException { final int slash = cases.indexOf('/'); final ScaleSuites.Suite suite = ScaleSuites.load(cases.substring(0, slash)); - final DDEvaluator evaluator = - new DDEvaluator( - () -> {}, - AgentBridge.NONE, - Settings.load(AgentBridge.NONE), - ExperimentationOptOutMode.DISABLED); - evaluator.accept(UniversalFlagConfigParser.parse(suite.config())); - provider = new Provider(new Provider.Options(), evaluator, false); - - final List loaded = new ArrayList<>(); - for (final Object json : suite.cases(cases.substring(slash + 1))) { - loaded.add(new ScaleCase((Map) json)); - } + provider = suite.provider(); + final List loaded = suite.cases(cases.substring(slash + 1)); for (int i = 0; i < loaded.size(); i++) { final ScaleCase scaleCase = loaded.get(i); scaleCase.check(scaleCase.evaluate(provider)); @@ -115,6 +110,9 @@ public void setUp() throws IOException { @TearDown(Level.Trial) public void tearDown() { + for (final ScaleCase scaleCase : scaleCases) { + scaleCase.check(scaleCase.evaluate(provider)); + } provider.shutdown(); } @@ -124,151 +122,4 @@ public ProviderEvaluation evaluate() { next = next + 1 == scaleCases.length ? 0 : next + 1; return scaleCase.evaluate(provider); } - - /** One evaluation case, with its expected value and reason. */ - private static final class ScaleCase { - private final String flag; - private final String variationType; - private final Object defaultValue; - private final Object expectedValue; - private final String expectedReason; - private final String targetingKey; - private final EvaluationContext context; - - ScaleCase(final Map json) { - this.flag = (String) json.get("flag"); - this.variationType = (String) json.get("variationType"); - final Map result = (Map) json.get("result"); - this.defaultValue = typed(json.get("defaultValue")); - this.expectedValue = result.get("value"); - this.expectedReason = (String) result.get("reason"); - this.targetingKey = (String) json.get("targetingKey"); - final MutableContext context = - new MutableContext(Value.objectToValue(json.get("attributes")).asStructure().asMap()); - context.setTargetingKey(targetingKey); - this.context = context; - } - - ProviderEvaluation evaluate(final Provider provider) { - switch (variationType) { - case "BOOLEAN": - return provider.getBooleanEvaluation(flag, (Boolean) defaultValue, context); - case "INTEGER": - return provider.getIntegerEvaluation(flag, (Integer) defaultValue, context); - case "NUMERIC": - return provider.getDoubleEvaluation(flag, (Double) defaultValue, context); - case "STRING": - return provider.getStringEvaluation(flag, (String) defaultValue, context); - case "JSON": - return provider.getObjectEvaluation(flag, (Value) defaultValue, context); - default: - throw new IllegalArgumentException("Unsupported variationType: " + variationType); - } - } - - void check(final ProviderEvaluation details) { - final Object actual = - details.getValue() instanceof Value - ? plain((Value) details.getValue()) - : details.getValue(); - if (!sameValue(expectedValue, actual) - || !Objects.equals(expectedReason, details.getReason())) { - throw new IllegalStateException( - "Unexpected result for " - + flag - + " (" - + targetingKey - + "): expected " - + expectedReason - + " " - + abbreviate(expectedValue) - + ", got " - + details.getReason() - + " " - + abbreviate(actual)); - } - } - - String describe() { - return flag + " (" + targetingKey + ") -> " + expectedReason; - } - - /** Converts a fixture value to the type that the provider takes for this variation type. */ - private Object typed(final Object value) { - if (value == null) { - return null; - } - switch (variationType) { - case "INTEGER": - return ((Number) value).intValue(); - case "NUMERIC": - return ((Number) value).doubleValue(); - case "JSON": - return Value.objectToValue(value); - default: - return value; - } - } - } - - /** Converts a returned value to maps, lists, strings, numbers, booleans and null. */ - private static Object plain(final Value value) { - if (value == null || value.isNull()) { - return null; - } - if (value.isStructure()) { - final Map map = new java.util.LinkedHashMap<>(); - value.asStructure().asMap().forEach((k, v) -> map.put(k, plain(v))); - return map; - } - if (value.isList()) { - final List list = new ArrayList<>(); - for (final Value item : value.asList()) { - list.add(plain(item)); - } - return list; - } - return value.asObject(); - } - - /** Equality that compares numbers by value, so 5 and 5.0 are the same. */ - private static boolean sameValue(final Object expected, final Object actual) { - if (expected instanceof Number && actual instanceof Number) { - return new java.math.BigDecimal(expected.toString()) - .compareTo(new java.math.BigDecimal(actual.toString())) - == 0; - } - if (expected instanceof Map && actual instanceof Map) { - final Map e = (Map) expected; - final Map a = (Map) actual; - if (e.size() != a.size()) { - return false; - } - for (final Map.Entry entry : e.entrySet()) { - if (!a.containsKey(entry.getKey()) || !sameValue(entry.getValue(), a.get(entry.getKey()))) { - return false; - } - } - return true; - } - if (expected instanceof List && actual instanceof List) { - final List e = (List) expected; - final List a = (List) actual; - if (e.size() != a.size()) { - return false; - } - for (int i = 0; i < e.size(); i++) { - if (!sameValue(e.get(i), a.get(i))) { - return false; - } - } - return true; - } - return Objects.equals(expected, actual); - } - - private static String abbreviate(final Object value) { - final String text = String.valueOf(value); - return text.length() <= 200 ? text : text.substring(0, 200) + "... (" + text.length() + ")"; - } } diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java index acebb1a4dfb..6d047098cae 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java @@ -5,9 +5,11 @@ import com.datadog.openfeature.internal.ufc.ServerConfiguration; import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; +import de.thetaphi.forbiddenapis.SuppressForbidden; import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; +import java.lang.ref.Reference; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -23,11 +25,12 @@ /** * Configuration load cost of the opt-in scale suites: the time to parse each suite's {@code - * ufc-config.json}. Work that an evaluator moves to load time, such as indexes, compiled patterns - * or prebuilt JSON values, shows here. + * ufc-config.json}. Work that moves into the parser, such as indexes, compiled patterns or prebuilt + * JSON values, shows here. * *

The setup also prints the heap that a parsed configuration retains. It is the used heap after - * a full collection, with and without the parsed configuration, so it is approximate. + * a full collection, with and without the parsed configuration, as the smallest of three + * measurements, so it is approximate. * *

Run: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteLoadBenchmark * -Pjmh.profilers=gc}. @@ -46,11 +49,20 @@ public class ScaleSuiteLoadBenchmark { private byte[] config; @Setup(Level.Trial) + @SuppressForbidden public void setUp() throws IOException { config = ScaleSuites.load(suite).config(); - final long before = usedHeapAfterCollection(); + // The first parse initializes the parser, so it is left out of the measurement. ServerConfiguration retained = UniversalFlagConfigParser.parse(config); - final long after = usedHeapAfterCollection(); + long retainedBytes = Long.MAX_VALUE; + for (int i = 0; i < 3; i++) { + retained = null; + final long before = usedHeapAfterCollection(); + retained = UniversalFlagConfigParser.parse(config); + final long after = usedHeapAfterCollection(); + Reference.reachabilityFence(retained); + retainedBytes = Math.min(retainedBytes, after - before); + } System.out.println( "[" + suite @@ -59,9 +71,8 @@ public void setUp() throws IOException { + " KB, " + retained.flags.size() + " flags, retained heap about " - + Math.max(0, after - before) / 1024 + + retainedBytes / 1024 + " KB"); - retained = null; } @Benchmark diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuites.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuites.java deleted file mode 100644 index 9c3b3b5b421..00000000000 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuites.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.datadog.openfeature; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** Reads the opt-in scale suites of the ffe-system-test-data submodule. */ -final class ScaleSuites { - private static final String SUITES_PATH = - "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites"; - private static final JsonFactory JSON_FACTORY = new JsonFactory(); - - private ScaleSuites() {} - - /** One suite directory: its configuration and its case files. */ - static final class Suite { - private final Path directory; - - private Suite(final Path directory) { - this.directory = directory; - } - - byte[] config() throws IOException { - return Files.readAllBytes(directory.resolve("ufc-config.json")); - } - - /** The cases of {@code cases/.json}. */ - List cases(final String name) throws IOException { - return (List) read(directory.resolve("cases").resolve(name + ".json")); - } - } - - static Suite load(final String name) { - Path directory = Paths.get("").toAbsolutePath(); - while (directory != null) { - final Path candidate = directory.resolve(SUITES_PATH).resolve(name); - if (Files.exists(candidate.resolve("ufc-config.json"))) { - return new Suite(candidate); - } - directory = directory.getParent(); - } - throw new IllegalStateException( - "Unable to find the FFE suite " + name + ": initialize the ffe-system-test-data submodule"); - } - - /** Reads a JSON file: integral numbers as Long, other numbers as Double. */ - private static Object read(final Path file) throws IOException { - try (JsonParser parser = JSON_FACTORY.createParser(Files.readAllBytes(file))) { - parser.nextToken(); - return readValue(parser); - } - } - - private static Object readValue(final JsonParser parser) throws IOException { - final JsonToken token = parser.currentToken(); - switch (token) { - case START_OBJECT: - final Map object = new LinkedHashMap<>(); - while (parser.nextToken() == JsonToken.FIELD_NAME) { - final String name = parser.currentName(); - parser.nextToken(); - object.put(name, readValue(parser)); - } - return object; - case START_ARRAY: - final List array = new ArrayList<>(); - while (parser.nextToken() != JsonToken.END_ARRAY) { - array.add(readValue(parser)); - } - return array; - case VALUE_STRING: - return parser.getText(); - case VALUE_NUMBER_INT: - return parser.getLongValue(); - case VALUE_NUMBER_FLOAT: - return parser.getDoubleValue(); - case VALUE_TRUE: - return Boolean.TRUE; - case VALUE_FALSE: - return Boolean.FALSE; - case VALUE_NULL: - return null; - default: - throw new IOException("Unexpected JSON token: " + token); - } - } -} diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java new file mode 100644 index 00000000000..8a3136ae281 --- /dev/null +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java @@ -0,0 +1,168 @@ +package com.datadog.openfeature; + +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.Value; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * One case of a scale suite of the shared FFE fixtures, with its expected value and reason. + * + *

JSON values are compared by numeric value: a JSON integer can come back as an integer or as a + * double, so that one check runs on every evaluator. {@code CanonicalFixtureTest} checks the types. + */ +final class ScaleCase { + private final String flag; + private final String variationType; + private final Object defaultValue; + private final Object expectedValue; + private final String expectedReason; + private final String targetingKey; + private final EvaluationContext context; + + ScaleCase(final Map json) { + this.flag = (String) json.get("flag"); + this.variationType = (String) json.get("variationType"); + final Map result = (Map) json.get("result"); + if (!Arrays.asList("value", "reason").containsAll(result.keySet())) { + throw new IllegalStateException( + "Unhandled expectations for " + json.get("flag") + ": " + result.keySet()); + } + this.defaultValue = typed(json.get("defaultValue")); + this.expectedValue = result.get("value"); + this.expectedReason = (String) result.get("reason"); + this.targetingKey = (String) json.get("targetingKey"); + final MutableContext context = + new MutableContext(Value.objectToValue(json.get("attributes")).asStructure().asMap()); + context.setTargetingKey(targetingKey); + this.context = context; + } + + ProviderEvaluation evaluate(final Provider provider) { + switch (variationType) { + case "BOOLEAN": + return provider.getBooleanEvaluation(flag, (Boolean) defaultValue, context); + case "INTEGER": + return provider.getIntegerEvaluation(flag, (Integer) defaultValue, context); + case "NUMERIC": + return provider.getDoubleEvaluation(flag, (Double) defaultValue, context); + case "STRING": + return provider.getStringEvaluation(flag, (String) defaultValue, context); + case "JSON": + return provider.getObjectEvaluation(flag, (Value) defaultValue, context); + default: + throw new IllegalArgumentException("Unsupported variationType: " + variationType); + } + } + + void check(final ProviderEvaluation details) { + final Object actual = + details.getValue() instanceof Value + ? plain((Value) details.getValue()) + : details.getValue(); + if (!sameValue(expectedValue, actual) + || !Objects.equals(expectedReason, details.getReason()) + || details.getErrorCode() != null) { + throw new IllegalStateException( + "Unexpected result for " + + flag + + " (" + + targetingKey + + "): expected " + + expectedReason + + " " + + abbreviate(expectedValue) + + ", got " + + details.getReason() + + " " + + abbreviate(actual)); + } + } + + String describe() { + return flag + " (" + targetingKey + ") -> " + expectedReason; + } + + /** Converts a fixture value to the type that the provider takes for this variation type. */ + private Object typed(final Object value) { + if (value == null) { + return null; + } + switch (variationType) { + case "INTEGER": + return ((Number) value).intValue(); + case "NUMERIC": + return ((Number) value).doubleValue(); + case "JSON": + return Value.objectToValue(value); + default: + return value; + } + } + + /** Converts a returned value to maps, lists, strings, numbers, booleans and null. */ + private static Object plain(final Value value) { + if (value == null || value.isNull()) { + return null; + } + if (value.isStructure()) { + final Map map = new LinkedHashMap<>(); + value.asStructure().asMap().forEach((k, v) -> map.put(k, plain(v))); + return map; + } + if (value.isList()) { + final List list = new ArrayList<>(); + for (final Value item : value.asList()) { + list.add(plain(item)); + } + return list; + } + return value.asObject(); + } + + /** Equality that compares numbers by value, so 5 and 5.0 are the same. */ + private static boolean sameValue(final Object expected, final Object actual) { + if (expected instanceof Number && actual instanceof Number) { + return new BigDecimal(expected.toString()).compareTo(new BigDecimal(actual.toString())) == 0; + } + if (expected instanceof Map && actual instanceof Map) { + final Map e = (Map) expected; + final Map a = (Map) actual; + if (e.size() != a.size()) { + return false; + } + for (final Map.Entry entry : e.entrySet()) { + if (!a.containsKey(entry.getKey()) || !sameValue(entry.getValue(), a.get(entry.getKey()))) { + return false; + } + } + return true; + } + if (expected instanceof List && actual instanceof List) { + final List e = (List) expected; + final List a = (List) actual; + if (e.size() != a.size()) { + return false; + } + for (int i = 0; i < e.size(); i++) { + if (!sameValue(e.get(i), a.get(i))) { + return false; + } + } + return true; + } + return Objects.equals(expected, actual); + } + + private static String abbreviate(final Object value) { + final String text = String.valueOf(value); + return text.length() <= 200 ? text : text.substring(0, 200) + "... (" + text.length() + ")"; + } +} diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java new file mode 100644 index 00000000000..eff02a68ff6 --- /dev/null +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java @@ -0,0 +1,75 @@ +package com.datadog.openfeature; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Evaluates every case of the opt-in scale suites of the shared FFE fixtures, as {@code + * CanonicalFixtureTest} does for the core cases. The scale benchmarks run the same check, so a + * change that resolves a large configuration wrongly fails here instead of only in a manual run. + */ +class ScaleSuiteTest { + private static final List SUITES = Arrays.asList("scale", "scale-json"); + + /** The case files that {@code ScaleSuiteBenchmark} lists in its {@code cases} parameter. */ + private static final List BENCHMARKED_CASE_FILES = + Arrays.asList( + "scale/flags", + "scale/numeric", + "scale/one-of", + "scale/one-of-many", + "scale/regex", + "scale/regex-many", + "scale/rules", + "scale/rules-sweep", + "scale/semver", + "scale/splits", + "scale/time-windows", + "scale/variations", + "scale/wide-context", + "scale-json/json-shapes", + "scale-json/json-sizes", + "scale-json/json-values"); + + static Stream caseFiles() throws IOException { + final List caseFiles = new ArrayList<>(); + for (final String suite : SUITES) { + for (final String file : ScaleSuites.load(suite).caseFiles()) { + caseFiles.add(suite + "/" + file); + } + } + return caseFiles.stream(); + } + + @Test + void benchmarkListsEveryCaseFile() throws IOException { + final List found = new ArrayList<>(); + caseFiles().forEach(found::add); + assertEquals(BENCHMARKED_CASE_FILES, found); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("caseFiles") + void evaluatesEveryCase(final String caseFile) throws IOException { + final int slash = caseFile.indexOf('/'); + final ScaleSuites.Suite suite = ScaleSuites.load(caseFile.substring(0, slash)); + final Provider provider = suite.provider(); + try { + for (final ScaleCase scaleCase : suite.cases(caseFile.substring(slash + 1))) { + // Twice: a cache or index built on the first evaluation must give the same result. + scaleCase.check(scaleCase.evaluate(provider)); + scaleCase.check(scaleCase.evaluate(provider)); + } + } finally { + provider.shutdown(); + } + } +} diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java new file mode 100644 index 00000000000..3a454e733d6 --- /dev/null +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java @@ -0,0 +1,93 @@ +package com.datadog.openfeature; + +import com.datadog.openfeature.internal.JsonReading; +import com.datadog.openfeature.internal.bridge.v1.AgentBridge; +import com.datadog.openfeature.internal.config.Settings; +import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Reads the opt-in scale suites of the ffe-system-test-data submodule, for {@code ScaleSuiteTest} + * and the scale benchmarks. + */ +final class ScaleSuites { + private static final String SUITES_PATH = + "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites"; + + private ScaleSuites() {} + + /** One suite directory: its configuration and its case files. */ + static final class Suite { + private final Path directory; + + private Suite(final Path directory) { + this.directory = directory; + } + + byte[] config() throws IOException { + return Files.readAllBytes(directory.resolve("ufc-config.json")); + } + + /** A provider that evaluates against this suite's configuration. */ + Provider provider() throws IOException { + final DDEvaluator evaluator = + new DDEvaluator( + () -> {}, + AgentBridge.NONE, + Settings.load(AgentBridge.NONE), + ExperimentationOptOutMode.DISABLED); + evaluator.accept(UniversalFlagConfigParser.parse(config())); + return new Provider(new Provider.Options(), evaluator, false); + } + + /** The names of the case files, without the {@code .json} extension, in sorted order. */ + List caseFiles() throws IOException { + final List names = new ArrayList<>(); + try (DirectoryStream files = + Files.newDirectoryStream(directory.resolve("cases"), "*.json")) { + for (final Path file : files) { + final String name = file.getFileName().toString(); + names.add(name.substring(0, name.length() - ".json".length())); + } + } + Collections.sort(names); + return names; + } + + /** The cases of {@code cases/.json}. */ + List cases(final String name) throws IOException { + final List cases = new ArrayList<>(); + for (final Object json : + (List) + JsonReading.read( + Files.readAllBytes(directory.resolve("cases").resolve(name + ".json")))) { + cases.add(new ScaleCase((Map) json)); + } + if (cases.isEmpty()) { + throw new IllegalStateException("The case file " + name + " has no cases"); + } + return cases; + } + } + + static Suite load(final String name) { + Path directory = Paths.get("").toAbsolutePath(); + while (directory != null) { + final Path candidate = directory.resolve(SUITES_PATH).resolve(name); + if (Files.exists(candidate.resolve("ufc-config.json"))) { + return new Suite(candidate); + } + directory = directory.getParent(); + } + throw new IllegalStateException( + "Unable to find the FFE suite " + name + ": initialize the ffe-system-test-data submodule"); + } +} From 09289ff1d948237ef2bc9ed6f8b39766b7dccd75 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 17 Nov 2026 04:11:33 +0000 Subject: [PATCH 04/18] Pin the fixtures to the latest head of the scale suite PR The fixture PR now checks that the committed scale files match the generator and fixes its docs. The case and configuration files are unchanged, so the benchmark numbers do not change. Co-Authored-By: Claude Opus 5.5 --- .../openfeature/src/test/resources/ffe-system-test-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index af0188a2b4f..f425d578dcf 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit af0188a2b4ff08ae6f9740ca89c904b812750ba7 +Subproject commit f425d578dcf04f61a49d9066ad525731d0158936 From 30df99975319dc9e5f7d1035401be17290fd08d9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 17 Nov 2026 04:39:13 +0000 Subject: [PATCH 05/18] Benchmark one-of lists of small numbers The fixtures add a case file with ONE_OF lists of the integers 1 to 1000 and of the powers of two up to 2^52, which an index keyed on exact doubles must still match. Pin them and benchmark the file; its 8 cases pass on master. Co-Authored-By: Claude Opus 5.5 --- .../jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java | 1 + .../src/test/java/com/datadog/openfeature/ScaleSuiteTest.java | 1 + .../openfeature/src/test/resources/ffe-system-test-data | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index b0ad49fccb9..23b741f0887 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -67,6 +67,7 @@ public class ScaleSuiteBenchmark { "scale/numeric", "scale/one-of", "scale/one-of-many", + "scale/one-of-small-numbers", "scale/regex", "scale/regex-many", "scale/rules", diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java index eff02a68ff6..274bd507764 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java @@ -26,6 +26,7 @@ class ScaleSuiteTest { "scale/numeric", "scale/one-of", "scale/one-of-many", + "scale/one-of-small-numbers", "scale/regex", "scale/regex-many", "scale/rules", diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index f425d578dcf..08e86993010 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit f425d578dcf04f61a49d9066ad525731d0158936 +Subproject commit 08e869930102e020a1a4479cfa79aa30be1abe31 From 8d0f39c5cf6376e4ba3bc09728a3229a0afd9b9a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 17 Nov 2026 04:46:49 +0000 Subject: [PATCH 06/18] Benchmark the scale cases with the client's layered context The OpenFeature client passes the provider a new LayeredEvaluationContext on each call, while the benchmark reused one MutableContext. A rule on `id` when the context has no `id` reads the layered context's key set, which it builds again on each call, so the benchmark did not show what applications pay for it. evaluateLayered wraps each case's context the way the client does, as ExperimentationOptOutBenchmark.evaluateFreshLayeredContext does. Co-Authored-By: Claude Opus 5.5 --- .../openfeature/ScaleSuiteBenchmark.java | 21 +++++++++++++++++-- .../com/datadog/openfeature/ScaleCase.java | 9 ++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index 23b741f0887..4f89fd794d1 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -4,6 +4,8 @@ import static java.util.concurrent.TimeUnit.SECONDS; import de.thetaphi.forbiddenapis.SuppressForbidden; +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.LayeredEvaluationContext; import dev.openfeature.sdk.ProviderEvaluation; import java.io.IOException; import java.util.List; @@ -38,8 +40,12 @@ * numeric value: a JSON integer can come back as an integer or as a double. This check guards the * measurement and is not the conformance check: {@code CanonicalFixtureTest} checks the types. * - *

The provider is called directly with a reused context: the OpenFeature client, its hooks and - * exposure logging are not measured. + *

The provider is called directly: the OpenFeature client, its hooks and exposure logging are + * not measured. {@code evaluate} passes a reused context. {@code evaluateLayered} passes a fresh + * {@code LayeredEvaluationContext} around it on each call, as the client does, so it also shows the + * cost of reading attributes through the client's context, such as a rule on {@code id} when the + * context has no {@code id}. Select one method with {@code + * -Pjmh.includes=ScaleSuiteBenchmark.evaluate$}. * *

The cases and their check are shared with {@code ScaleSuiteTest}, which runs them in the unit * tests. One fork is enough for allocation and for large time differences. For a time difference @@ -123,4 +129,15 @@ public ProviderEvaluation evaluate() { next = next + 1 == scaleCases.length ? 0 : next + 1; return scaleCase.evaluate(provider); } + + /** As {@link #evaluate}, with the fresh layered context that the OpenFeature client builds. */ + @Benchmark + public ProviderEvaluation evaluateLayered() { + final ScaleCase scaleCase = scaleCases[next]; + next = next + 1 == scaleCases.length ? 0 : next + 1; + return scaleCase.evaluate( + provider, + new LayeredEvaluationContext( + ImmutableContext.EMPTY, ImmutableContext.EMPTY, null, scaleCase.context())); + } } diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java index 8a3136ae281..61d317c75b8 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java @@ -46,6 +46,11 @@ final class ScaleCase { } ProviderEvaluation evaluate(final Provider provider) { + return evaluate(provider, context); + } + + /** Evaluates this case with another context, such as a layered one wrapping {@link #context}. */ + ProviderEvaluation evaluate(final Provider provider, final EvaluationContext context) { switch (variationType) { case "BOOLEAN": return provider.getBooleanEvaluation(flag, (Boolean) defaultValue, context); @@ -86,6 +91,10 @@ void check(final ProviderEvaluation details) { } } + EvaluationContext context() { + return context; + } + String describe() { return flag + " (" + targetingKey + ") -> " + expectedReason; } From 72d60caec00785e1f179bd306d254dd0812f2cad Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 17 Nov 2026 05:15:23 +0000 Subject: [PATCH 07/18] Pin the fixtures with a 4 MB largest JSON value The fixtures replace the 10 MB JSON flag with a 4 MB one, which brings the scale-json configuration from 14 MB down to 8 MB. The other cases are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java | 2 +- .../openfeature/src/test/resources/ffe-system-test-data | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index 4f89fd794d1..d3f591b3362 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -26,7 +26,7 @@ /** * Evaluation cost on the opt-in scale suites of the shared FFE fixtures ({@code * evaluation-suites/scale} and {@code evaluation-suites/scale-json}): large lists, regex, rules, - * splits and shards, time windows, many variations and flags, and JSON values up to 10 MB. The + * splits and shards, time windows, many variations and flags, and JSON values up to 4 MB. The * dimensions are listed in {@code evaluation-suites/scale/AGENTS.md} of the fixtures. * *

Each operation evaluates a case of the selected case file through the provider. With {@code diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index 08e86993010..ec0ec0591f0 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit 08e869930102e020a1a4479cfa79aa30be1abe31 +Subproject commit ec0ec0591f0b6b1bce35aa4cdc195f9652a0a61d From e69a0af82572055ca618047c57082a171aa8eb45 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 17 Nov 2026 23:09:39 +0000 Subject: [PATCH 08/18] Give each benchmark thread its own case cursor With several JMH threads and index=all, the threads shared one unsynchronized cursor, so a thread could advance it past the last case and fail the run with ArrayIndexOutOfBoundsException. The cursor is now thread-scoped state, and the provider and cases stay shared. Co-Authored-By: Claude Opus 5.5 --- .../openfeature/ScaleSuiteBenchmark.java | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index d3f591b3362..c74a71d61ce 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -30,9 +30,9 @@ * dimensions are listed in {@code evaluation-suites/scale/AGENTS.md} of the fixtures. * *

Each operation evaluates a case of the selected case file through the provider. With {@code - * index=all}, the operations go through the cases of the file in turn, so a row is the unweighted - * mean over every case of the file, and its slowest case dominates it. Select one case to measure - * one shape: the setup prints the index of each case. + * index=all}, each benchmark thread goes through the cases of the file in turn, so a row is the + * unweighted mean over every case of the file, and its slowest case dominates it. Select one case + * to measure one shape: the setup prints the index of each case. * *

The setup evaluates every case once and the teardown again after the measurement. Both fail if * a value or reason differs from the expected one, or if an evaluation reports an error, so a run @@ -95,7 +95,18 @@ public class ScaleSuiteBenchmark { private Provider provider; private ScaleCase[] scaleCases; - private int next; + + /** The position of one benchmark thread in the cases, so threads share no mutable state. */ + @State(Scope.Thread) + public static class Cursor { + private int next; + + ScaleCase next(final ScaleCase[] scaleCases) { + final ScaleCase scaleCase = scaleCases[next]; + next = next + 1 == scaleCases.length ? 0 : next + 1; + return scaleCase; + } + } @Setup(Level.Trial) @SuppressForbidden @@ -124,17 +135,14 @@ public void tearDown() { } @Benchmark - public ProviderEvaluation evaluate() { - final ScaleCase scaleCase = scaleCases[next]; - next = next + 1 == scaleCases.length ? 0 : next + 1; - return scaleCase.evaluate(provider); + public ProviderEvaluation evaluate(final Cursor cursor) { + return cursor.next(scaleCases).evaluate(provider); } /** As {@link #evaluate}, with the fresh layered context that the OpenFeature client builds. */ @Benchmark - public ProviderEvaluation evaluateLayered() { - final ScaleCase scaleCase = scaleCases[next]; - next = next + 1 == scaleCases.length ? 0 : next + 1; + public ProviderEvaluation evaluateLayered(final Cursor cursor) { + final ScaleCase scaleCase = cursor.next(scaleCases); return scaleCase.evaluate( provider, new LayeredEvaluationContext( From 911924fbc9bddc55adcc66ee25d292d3ec70d7cd Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 17 Nov 2026 23:24:39 +0000 Subject: [PATCH 09/18] Pin the fixtures with first-match cases and check selection metadata The pinned scale suite adds first-match cases and expects the selected serialId, doLog, and split features on scale-many-variations. ScaleCase now checks those fields from the flag metadata, as CanonicalFixtureTest does. The scale suites run without span enrichment, so an allocation that does not log carries no doLog key, and a missing key means false. Co-Authored-By: Claude Opus 5.5 --- .../openfeature/ScaleSuiteBenchmark.java | 9 +-- .../com/datadog/openfeature/ScaleCase.java | 65 ++++++++++++++++++- .../src/test/resources/ffe-system-test-data | 2 +- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index c74a71d61ce..c7bb9adcbbd 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -35,10 +35,11 @@ * to measure one shape: the setup prints the index of each case. * *

The setup evaluates every case once and the teardown again after the measurement. Both fail if - * a value or reason differs from the expected one, or if an evaluation reports an error, so a run - * that resolves a case wrongly fails instead of reporting a number. JSON values are compared by - * numeric value: a JSON integer can come back as an integer or as a double. This check guards the - * measurement and is not the conformance check: {@code CanonicalFixtureTest} checks the types. + * a value, reason, or expected selection metadata (serial ID, {@code doLog}, split features) + * differs from the expected one, or if an evaluation reports an error, so a run that resolves a + * case wrongly fails instead of reporting a number. JSON values are compared by numeric value: a + * JSON integer can come back as an integer or as a double. This check guards the measurement and is + * not the conformance check: {@code CanonicalFixtureTest} checks the types. * *

The provider is called directly: the OpenFeature client, its hooks and exposure logging are * not measured. {@code evaluate} passes a reused context. {@code evaluateLayered} passes a fresh diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java index 61d317c75b8..bc84cfac140 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java @@ -1,5 +1,11 @@ package com.datadog.openfeature; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.DO_LOG; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVALUATION_FEATURE_PREFIX; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_FEATURE_PREFIX; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.FEATURE_PREFIX; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.SPLIT_SERIAL_ID; + import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.ProviderEvaluation; @@ -13,7 +19,8 @@ import java.util.Objects; /** - * One case of a scale suite of the shared FFE fixtures, with its expected value and reason. + * One case of a scale suite of the shared FFE fixtures, with its expected value and reason, and the + * selected {@code serialId}, {@code doLog}, and split features when the case expects them. * *

JSON values are compared by numeric value: a JSON integer can come back as an integer or as a * double, so that one check runs on every evaluator. {@code CanonicalFixtureTest} checks the types. @@ -24,6 +31,10 @@ final class ScaleCase { private final Object defaultValue; private final Object expectedValue; private final String expectedReason; + private final boolean checksSerialId; + private final Number expectedSerialId; + private final Boolean expectedDoLog; + private final Map expectedFeatures; private final String targetingKey; private final EvaluationContext context; @@ -31,13 +42,18 @@ final class ScaleCase { this.flag = (String) json.get("flag"); this.variationType = (String) json.get("variationType"); final Map result = (Map) json.get("result"); - if (!Arrays.asList("value", "reason").containsAll(result.keySet())) { + if (!Arrays.asList("value", "reason", "serialId", "doLog", "features") + .containsAll(result.keySet())) { throw new IllegalStateException( "Unhandled expectations for " + json.get("flag") + ": " + result.keySet()); } this.defaultValue = typed(json.get("defaultValue")); this.expectedValue = result.get("value"); this.expectedReason = (String) result.get("reason"); + this.checksSerialId = result.containsKey("serialId"); + this.expectedSerialId = (Number) result.get("serialId"); + this.expectedDoLog = (Boolean) result.get("doLog"); + this.expectedFeatures = (Map) result.get("features"); this.targetingKey = (String) json.get("targetingKey"); final MutableContext context = new MutableContext(Value.objectToValue(json.get("attributes")).asStructure().asMap()); @@ -89,6 +105,51 @@ void check(final ProviderEvaluation details) { + " " + abbreviate(actual)); } + final Map metadata = details.getFlagMetadata().asUnmodifiableMap(); + if (checksSerialId) { + final Object serialId = metadata.get(SPLIT_SERIAL_ID); + if (expectedSerialId == null ? serialId != null : !sameValue(expectedSerialId, serialId)) { + throw unexpectedMetadata("serialId", expectedSerialId, serialId); + } + } + // Without span enrichment, as the scale suites run, an allocation that does not log carries no + // doLog key and no serial ID, so a missing doLog means false. + final Object doLog = metadata.getOrDefault(DO_LOG, false); + if (expectedDoLog != null && !expectedDoLog.equals(doLog)) { + throw unexpectedMetadata("doLog", expectedDoLog, doLog); + } + if (expectedFeatures != null) { + final Map features = new LinkedHashMap<>(); + features.put( + "HOOK", + SplitFeatureMetadata.featuresWithPrefix(details.getFlagMetadata(), FEATURE_PREFIX)); + features.put( + "EXPOSURE", + SplitFeatureMetadata.featuresWithPrefix( + details.getFlagMetadata(), EXPOSURE_FEATURE_PREFIX)); + features.put( + "EVALUATION", + SplitFeatureMetadata.featuresWithPrefix( + details.getFlagMetadata(), EVALUATION_FEATURE_PREFIX)); + if (!sameValue(expectedFeatures, features)) { + throw unexpectedMetadata("features", expectedFeatures, features); + } + } + } + + private IllegalStateException unexpectedMetadata( + final String field, final Object expected, final Object actual) { + return new IllegalStateException( + "Unexpected " + + field + + " for " + + flag + + " (" + + targetingKey + + "): expected " + + abbreviate(expected) + + ", got " + + abbreviate(actual)); } EvaluationContext context() { diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index ec0ec0591f0..52205a7155f 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit ec0ec0591f0b6b1bce35aa4cdc195f9652a0a61d +Subproject commit 52205a7155f9c28c5bdd29b5897478e789035bfd From 89a60359a4c11732fb8e80c36375f7943773c450 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 18 Nov 2026 02:21:46 +0000 Subject: [PATCH 10/18] Pin the fixtures to the merged scale suites DataDog/ffe-system-test-data#78 merged as 508a342 with the same suites as its head. Main also brings mutually exclusive layer cases and precomputed fixture corrections, which the unit and smoke tests pass. Co-Authored-By: Claude Opus 5.5 --- .../openfeature/src/test/resources/ffe-system-test-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index 52205a7155f..508a3425517 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit 52205a7155f9c28c5bdd29b5897478e789035bfd +Subproject commit 508a3425517b40b4617f09771a3a7e7a8fd76344 From 89120c71fea7424f73e5bb8ed1f0010f9689f41b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 18 Nov 2026 15:49:07 +0000 Subject: [PATCH 11/18] Move the shared fixtures into dd-openfeature The SDK is the main reader of the ffe-system-test-data submodule and the only reader of the vendored experimentation opt-out cases, so both move from the smoke-test resources to dd-openfeature/src/test/fixtures. They stay outside any source set: the tests and benchmarks read them from disk, and the smoke tests copy ufc-config.json and evaluation-cases into their resources. The jmh task declares the scale suites as a file tree, which is empty rather than missing when the submodule is not initialized, so a jmh run on such a checkout no longer fails before it starts. Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 2 +- .gitmodules | 4 +-- BUILDING.md | 2 +- dd-openfeature/build.gradle.kts | 27 +++++++------------ .../experimentation-opt-out/README.md | 0 .../experimentation-opt-out/cases.json | 0 .../experimentation-opt-out/verify.sh | 0 .../src/test/fixtures}/ffe-system-test-data | 0 .../openfeature/CanonicalFixtureTest.java | 2 +- .../ExperimentationOptOutProviderTest.java | 2 +- .../com/datadog/openfeature/ScaleSuites.java | 2 +- .../delivery/FlagSelectionVectorsTest.java | 2 +- dd-smoke-tests/openfeature/build.gradle | 8 ++++++ 13 files changed, 26 insertions(+), 25 deletions(-) rename {dd-smoke-tests/openfeature/src/test/resources => dd-openfeature/src/test/fixtures}/experimentation-opt-out/README.md (100%) rename {dd-smoke-tests/openfeature/src/test/resources => dd-openfeature/src/test/fixtures}/experimentation-opt-out/cases.json (100%) rename {dd-smoke-tests/openfeature/src/test/resources => dd-openfeature/src/test/fixtures}/experimentation-opt-out/verify.sh (100%) rename {dd-smoke-tests/openfeature/src/test/resources => dd-openfeature/src/test/fixtures}/ffe-system-test-data (100%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d610f967de5..74e97fc551d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -42,7 +42,7 @@ updates: schedule: interval: weekly allow: - - dependency-name: dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data + - dependency-name: dd-openfeature/src/test/fixtures/ffe-system-test-data labels: - 'comp: openfeature' - 'tag: dependencies' diff --git a/.gitmodules b/.gitmodules index 18769cc41ae..43d8c101a96 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "dd-java-agent/agent-jmxfetch/integrations-core"] path = dd-java-agent/agent-jmxfetch/integrations-core url = https://github.com/DataDog/integrations-core.git -[submodule "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data"] - path = dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +[submodule "dd-openfeature/src/test/fixtures/ffe-system-test-data"] + path = dd-openfeature/src/test/fixtures/ffe-system-test-data url = https://github.com/DataDog/ffe-system-test-data.git diff --git a/BUILDING.md b/BUILDING.md index 0ea6bfcec8a..5f7d4d1d81a 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -180,7 +180,7 @@ winget install --id Docker.DockerDesktop > [!TIP] > This will keep the submodules in `dd-java-agent/agent-jmxfetch/integrations-core` and -> `dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data` up-to-date. +> `dd-openfeature/src/test/fixtures/ffe-system-test-data` up-to-date. > There is also an automated check when opening a pull request if you are trying to submit a module version change (usually an outdated version). > [!NOTE] diff --git a/dd-openfeature/build.gradle.kts b/dd-openfeature/build.gradle.kts index 65835411e02..fab01f4d8ff 100644 --- a/dd-openfeature/build.gradle.kts +++ b/dd-openfeature/build.gradle.kts @@ -147,25 +147,18 @@ tasks.named("check") { dependsOn(japicmpPublicApi) } -// These suites read shared fixtures directly from the smoke-test resources. +// The shared FFE fixtures live outside any source set, so Gradle neither copies nor packages them: +// the tests and benchmarks read them from disk, and the smoke tests copy the files they need. +val fixtures = layout.projectDirectory.dir("src/test/fixtures") + tasks.withType().configureEach { - inputs.dir( - rootProject.layout.projectDirectory.dir( - "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data" - ) - ).withPathSensitivity(PathSensitivity.RELATIVE) - inputs.dir( - rootProject.layout.projectDirectory.dir( - "dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out" - ) - ).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.dir(fixtures).withPropertyName("fixtures").withPathSensitivity(PathSensitivity.RELATIVE) } -// The scale benchmarks read the fixtures from disk at run time, so a fixture change reruns them. +// The scale benchmarks read the fixtures from disk at run time, so a fixture change reruns them. A +// file tree rather than a directory: the suites are missing when the submodule is not initialized. tasks.named("jmh") { - inputs.dir( - rootProject.layout.projectDirectory.dir( - "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites" - ) - ).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.files( + fixtures.dir("ffe-system-test-data").asFileTree.matching { include("evaluation-suites/**") } + ).withPropertyName("scaleSuites").withPathSensitivity(PathSensitivity.RELATIVE) } diff --git a/dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out/README.md b/dd-openfeature/src/test/fixtures/experimentation-opt-out/README.md similarity index 100% rename from dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out/README.md rename to dd-openfeature/src/test/fixtures/experimentation-opt-out/README.md diff --git a/dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out/cases.json b/dd-openfeature/src/test/fixtures/experimentation-opt-out/cases.json similarity index 100% rename from dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out/cases.json rename to dd-openfeature/src/test/fixtures/experimentation-opt-out/cases.json diff --git a/dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out/verify.sh b/dd-openfeature/src/test/fixtures/experimentation-opt-out/verify.sh similarity index 100% rename from dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out/verify.sh rename to dd-openfeature/src/test/fixtures/experimentation-opt-out/verify.sh diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-openfeature/src/test/fixtures/ffe-system-test-data similarity index 100% rename from dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data rename to dd-openfeature/src/test/fixtures/ffe-system-test-data diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/CanonicalFixtureTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/CanonicalFixtureTest.java index 787fefa1c11..0ca20613cea 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/CanonicalFixtureTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/CanonicalFixtureTest.java @@ -48,7 +48,7 @@ */ class CanonicalFixtureTest { private static final String CANONICAL_FIXTURE_PATH = - "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data"; + "dd-openfeature/src/test/fixtures/ffe-system-test-data"; @Test void canonicalFixturesArePresent() throws IOException { diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ExperimentationOptOutProviderTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ExperimentationOptOutProviderTest.java index 041aa34e366..b3109be0089 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/ExperimentationOptOutProviderTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ExperimentationOptOutProviderTest.java @@ -483,7 +483,7 @@ private static Path fixture(final String name) { while (directory != null) { final Path candidate = directory - .resolve("dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out") + .resolve("dd-openfeature/src/test/fixtures/experimentation-opt-out") .resolve(name); if (Files.isRegularFile(candidate)) { return candidate; diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java index 3a454e733d6..108f481aa72 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java @@ -20,7 +20,7 @@ */ final class ScaleSuites { private static final String SUITES_PATH = - "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites"; + "dd-openfeature/src/test/fixtures/ffe-system-test-data/evaluation-suites"; private ScaleSuites() {} diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/delivery/FlagSelectionVectorsTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/delivery/FlagSelectionVectorsTest.java index 12f5460395e..0fee27eb767 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/delivery/FlagSelectionVectorsTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/delivery/FlagSelectionVectorsTest.java @@ -30,7 +30,7 @@ */ class FlagSelectionVectorsTest { private static final String VECTORS_PATH = - "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/flag-selection/selection-vectors.json"; + "dd-openfeature/src/test/fixtures/ffe-system-test-data/flag-selection/selection-vectors.json"; private static final URI ENDPOINT = URI.create( "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server?dd_env=prod"); diff --git a/dd-smoke-tests/openfeature/build.gradle b/dd-smoke-tests/openfeature/build.gradle index f085174d280..2d289e1a167 100644 --- a/dd-smoke-tests/openfeature/build.gradle +++ b/dd-smoke-tests/openfeature/build.gradle @@ -22,6 +22,14 @@ dependencies { testImplementation project(':dd-smoke-tests:smoke-test-common') } +// The shared FFE fixtures belong to dd-openfeature: copy the files these tests read. +tasks.named('processTestResources', ProcessResources) { + from(rootProject.file('dd-openfeature/src/test/fixtures')) { + include 'ffe-system-test-data/ufc-config.json' + include 'ffe-system-test-data/evaluation-cases/**' + } +} + spotless { java { target "**/*.java" From 6f3c1dd2693c047193a6e48d3c07a6b2b650051d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 18 Nov 2026 15:49:07 +0000 Subject: [PATCH 12/18] Ignore the JVM settings in the scale suites The suites read the system properties and environment, so setting DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED turned span enrichment on: the benchmarks measured the enriched metadata path and the expected selection metadata no longer described it. Co-Authored-By: Claude Opus 5.5 --- .../java/com/datadog/openfeature/ScaleSuites.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java index 108f481aa72..b25a90ca338 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java @@ -2,7 +2,7 @@ import com.datadog.openfeature.internal.JsonReading; import com.datadog.openfeature.internal.bridge.v1.AgentBridge; -import com.datadog.openfeature.internal.config.Settings; +import com.datadog.openfeature.internal.config.TestSettings; import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; import java.io.IOException; import java.nio.file.DirectoryStream; @@ -36,14 +36,15 @@ byte[] config() throws IOException { return Files.readAllBytes(directory.resolve("ufc-config.json")); } - /** A provider that evaluates against this suite's configuration. */ + /** + * A provider that evaluates against this suite's configuration. Its settings ignore the JVM + * system properties and environment, so span enrichment stays off and the expected selection + * metadata holds. + */ Provider provider() throws IOException { final DDEvaluator evaluator = new DDEvaluator( - () -> {}, - AgentBridge.NONE, - Settings.load(AgentBridge.NONE), - ExperimentationOptOutMode.DISABLED); + () -> {}, AgentBridge.NONE, TestSettings.of(), ExperimentationOptOutMode.DISABLED); evaluator.accept(UniversalFlagConfigParser.parse(config())); return new Provider(new Provider.Options(), evaluator, false); } From 9b7f7de20f7e1ec0da7fc78b8e6adea4b1abd268 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 18 Nov 2026 15:49:07 +0000 Subject: [PATCH 13/18] Parse each scale suite once in the test Each case file parsed its suite again, 14 times for the 7.8 MB scale configuration. One provider per suite, shut down after all cases, parses each configuration once. Co-Authored-By: Claude Opus 5.5 --- .../datadog/openfeature/ScaleSuiteTest.java | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java index 274bd507764..ecffb2fc8a4 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java @@ -5,8 +5,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Stream; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -19,6 +22,9 @@ class ScaleSuiteTest { private static final List SUITES = Arrays.asList("scale", "scale-json"); + /** One provider per suite, so each configuration is parsed once. */ + private static final Map PROVIDERS = new HashMap<>(); + /** The case files that {@code ScaleSuiteBenchmark} lists in its {@code cases} parameter. */ private static final List BENCHMARKED_CASE_FILES = Arrays.asList( @@ -50,6 +56,12 @@ static Stream caseFiles() throws IOException { return caseFiles.stream(); } + @AfterAll + static void shutDownProviders() { + PROVIDERS.values().forEach(Provider::shutdown); + PROVIDERS.clear(); + } + @Test void benchmarkListsEveryCaseFile() throws IOException { final List found = new ArrayList<>(); @@ -61,16 +73,17 @@ void benchmarkListsEveryCaseFile() throws IOException { @MethodSource("caseFiles") void evaluatesEveryCase(final String caseFile) throws IOException { final int slash = caseFile.indexOf('/'); - final ScaleSuites.Suite suite = ScaleSuites.load(caseFile.substring(0, slash)); - final Provider provider = suite.provider(); - try { - for (final ScaleCase scaleCase : suite.cases(caseFile.substring(slash + 1))) { - // Twice: a cache or index built on the first evaluation must give the same result. - scaleCase.check(scaleCase.evaluate(provider)); - scaleCase.check(scaleCase.evaluate(provider)); - } - } finally { - provider.shutdown(); + final String name = caseFile.substring(0, slash); + final ScaleSuites.Suite suite = ScaleSuites.load(name); + Provider provider = PROVIDERS.get(name); + if (provider == null) { + provider = suite.provider(); + PROVIDERS.put(name, provider); + } + for (final ScaleCase scaleCase : suite.cases(caseFile.substring(slash + 1))) { + // Twice: a cache or index built on the first evaluation must give the same result. + scaleCase.check(scaleCase.evaluate(provider)); + scaleCase.check(scaleCase.evaluate(provider)); } } } From 109bd9f3f325db68c313913575367c15fa2cc3e7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 18 Nov 2026 15:49:07 +0000 Subject: [PATCH 14/18] Check the benchmark's own case-file list ScaleSuiteTest compared the fixtures with a copy of the benchmark's @Param list, so a case file missing from the benchmark went unnoticed. The test now reads the @Param by reflection, with the jmh classes on its runtime classpath. Co-Authored-By: Claude Opus 5.5 --- dd-openfeature/build.gradle.kts | 6 +++ .../openfeature/ScaleSuiteBenchmark.java | 3 +- .../datadog/openfeature/ScaleSuiteTest.java | 44 +++++++++---------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/dd-openfeature/build.gradle.kts b/dd-openfeature/build.gradle.kts index fab01f4d8ff..438eb783561 100644 --- a/dd-openfeature/build.gradle.kts +++ b/dd-openfeature/build.gradle.kts @@ -155,6 +155,12 @@ tasks.withType().configureEach { inputs.dir(fixtures).withPropertyName("fixtures").withPathSensitivity(PathSensitivity.RELATIVE) } +// ScaleSuiteTest checks the case files that ScaleSuiteBenchmark lists in its @Param. It reads the +// annotation by reflection, as the test source set cannot compile against the jmh one. +tasks.named("test") { + classpath += sourceSets["jmh"].output + configurations["jmhRuntimeClasspath"] +} + // The scale benchmarks read the fixtures from disk at run time, so a fixture change reruns them. A // file tree rather than a directory: the suites are missing when the submodule is not initialized. tasks.named("jmh") { diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index c7bb9adcbbd..52e7c655cca 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -67,7 +67,8 @@ public class ScaleSuiteBenchmark { /** * A case file, as {@code /} without the {@code .json} extension. {@code - * ScaleSuiteTest} fails when this list misses a case file of the fixtures. + * ScaleSuiteTest} reads this list and fails unless it names every case file of the fixtures, in + * sorted order. */ @Param({ "scale/flags", diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java index ecffb2fc8a4..a0c3b14f52c 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java @@ -3,6 +3,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -25,27 +27,6 @@ class ScaleSuiteTest { /** One provider per suite, so each configuration is parsed once. */ private static final Map PROVIDERS = new HashMap<>(); - /** The case files that {@code ScaleSuiteBenchmark} lists in its {@code cases} parameter. */ - private static final List BENCHMARKED_CASE_FILES = - Arrays.asList( - "scale/flags", - "scale/numeric", - "scale/one-of", - "scale/one-of-many", - "scale/one-of-small-numbers", - "scale/regex", - "scale/regex-many", - "scale/rules", - "scale/rules-sweep", - "scale/semver", - "scale/splits", - "scale/time-windows", - "scale/variations", - "scale/wide-context", - "scale-json/json-shapes", - "scale-json/json-sizes", - "scale-json/json-values"); - static Stream caseFiles() throws IOException { final List caseFiles = new ArrayList<>(); for (final String suite : SUITES) { @@ -63,10 +44,10 @@ static void shutDownProviders() { } @Test - void benchmarkListsEveryCaseFile() throws IOException { + void benchmarkListsEveryCaseFile() throws ReflectiveOperationException, IOException { final List found = new ArrayList<>(); caseFiles().forEach(found::add); - assertEquals(BENCHMARKED_CASE_FILES, found); + assertEquals(benchmarkedCaseFiles(), found); } @ParameterizedTest(name = "{0}") @@ -86,4 +67,21 @@ void evaluatesEveryCase(final String caseFile) throws IOException { scaleCase.check(scaleCase.evaluate(provider)); } } + + /** + * The {@code @Param} values of {@code ScaleSuiteBenchmark.cases}. The test source set cannot + * compile against the jmh one, so the build adds the benchmark classes to this test's runtime + * classpath and the annotation is read by name. + */ + private static List benchmarkedCaseFiles() throws ReflectiveOperationException { + final Field cases = + Class.forName("com.datadog.openfeature.ScaleSuiteBenchmark").getField("cases"); + for (final Annotation annotation : cases.getAnnotations()) { + final Class type = annotation.annotationType(); + if ("org.openjdk.jmh.annotations.Param".equals(type.getName())) { + return Arrays.asList((String[]) type.getMethod("value").invoke(annotation)); + } + } + throw new AssertionError("ScaleSuiteBenchmark.cases has no @Param"); + } } From 971fcdf9c78e0b8bc3a85cbfe9c0ad922b9566bf Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 19 Nov 2026 01:06:17 +0000 Subject: [PATCH 15/18] Parenthesize the cursor wrap condition Co-Authored-By: Claude Opus 5.5 --- .../jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java index 52e7c655cca..4a7248bccd3 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -105,7 +105,7 @@ public static class Cursor { ScaleCase next(final ScaleCase[] scaleCases) { final ScaleCase scaleCase = scaleCases[next]; - next = next + 1 == scaleCases.length ? 0 : next + 1; + next = (next + 1 == scaleCases.length) ? 0 : next + 1; return scaleCase; } } From d27533f716322cbb694bd1d2bcdb8fb71cfc126a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 19 Nov 2026 03:12:10 +0000 Subject: [PATCH 16/18] Share flag metadata across evaluations of a split in the same millisecond FlagEvaluator built a new ImmutableMetadata on every successful evaluation, although everything in it but the timestamp is fixed by the split, and the timestamp has millisecond resolution. A fixed, direct-mapped, lock-free table now keeps the metadata built last for a key of the Split, Allocation and Flag references, the consent, the exposure state and the evaluation millisecond, so evaluations of the same split in the same millisecond share one immutable instance. It is a burst coalescer, not a cache: an entry is reusable only within its millisecond. Keying on references means a reloaded configuration never matches an entry of the previous one. Entries have only final fields and are published with a single reference store, so a lost race only costs a rebuild. DD_FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE sets the number of slots (1024 by default, rounded up to a power of two, at most 65536); 0 builds the metadata of every evaluation, as before. Co-authored-by: Douglas Q Hawkins Co-Authored-By: Claude Opus 5.5 --- dd-openfeature/README.md | 1 + .../EvaluationMetadataTableBenchmark.java | 173 +++++++++++ .../com/datadog/openfeature/DDEvaluator.java | 5 +- .../openfeature/internal/config/Settings.java | 28 +- .../evaluation/EvaluationMetadataTable.java | 182 +++++++++++ .../internal/evaluation/FlagEvaluator.java | 99 +++++- .../internal/config/SettingsTest.java | 16 + .../EvaluationMetadataTableTest.java | 294 ++++++++++++++++++ .../api/config/FeatureFlaggingConfig.java | 9 + metadata/supported-configurations.json | 8 + 10 files changed, 802 insertions(+), 13 deletions(-) create mode 100644 dd-openfeature/src/jmh/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableBenchmark.java create mode 100644 dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java create mode 100644 dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java diff --git a/dd-openfeature/README.md b/dd-openfeature/README.md index 60c735c7cb1..25e617ed640 100644 --- a/dd-openfeature/README.md +++ b/dd-openfeature/README.md @@ -54,6 +54,7 @@ Settings are read from the Datadog Java agent configuration when attached, then | `DD_FEATURE_FLAGS_EXPOSURES_DATADOG_LOGGING_ENABLED` | `true` | Sends exposures to Datadog. See [Stop sending exposures to Datadog](#stop-sending-exposures-to-datadog). | | `DD_FEATURE_FLAGS_EXPERIMENTATION_OPT_OUT_MODE` | `disabled` | Controls per-subject experimentation opt-out: `disabled`, `immediate`, or `future_experiments_only`. See [Per-subject experimentation opt-out](#per-subject-experimentation-opt-out). | | `DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED` | `false` | Lets the `dd_evaluation_telemetry_opt_out` context attribute opt subjects out of evaluation telemetry. See [Per-subject evaluation telemetry opt-out](#per-subject-evaluation-telemetry-opt-out). | +| `DD_FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE` | `1024` | Slots of the table that lets evaluations resolving the same split in the same millisecond share their flag metadata, rounded up to a power of two (at most 65536). `0` builds the metadata of every evaluation. | | `DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED` | `false` | Adds flag evaluations to the local root span. Requires `dd-java-agent`. | | `DD_API_KEY`, `DD_SITE`, `DD_ENV`, `DD_SERVICE`, `DD_VERSION` | | Credentials and event context. | | `DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED` | | Deprecated: `true` selects `remote_config`, `false` disables Feature Flags. An explicit source wins. | diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableBenchmark.java new file mode 100644 index 00000000000..83144bae84d --- /dev/null +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableBenchmark.java @@ -0,0 +1,173 @@ +package com.datadog.openfeature.internal.evaluation; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.datadog.openfeature.ExperimentationOptOutMode; +import com.datadog.openfeature.internal.ufc.ServerConfiguration; +import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; +import de.thetaphi.forbiddenapis.SuppressForbidden; +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.Value; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Locale; +import java.util.concurrent.atomic.LongAdder; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Concurrent requests that each evaluate the same {@code flagsPerRequest} flags of the 10,000-flag + * scale configuration ({@code evaluation-suites/scale}), for a subject and cohort that change on + * every request, on 12 threads sharing one evaluator. It shows what the evaluation metadata table + * saves when many threads resolve the same splits in the same millisecond, and whether its slot + * writes contend. {@code tableSize=0} builds the metadata of every evaluation, as without the + * table. + * + *

An operation is one request. Each measurement iteration prints the table's hit rate (the share + * of successful evaluations that reused metadata) and its misses per millisecond. The lowest + * possible miss rate is one per distinct key per millisecond: a request resolves {@code + * flagsPerRequest} splits, and a cohort can switch one of them, so about {@code flagsPerRequest + + * 1} keys. Misses above that come from threads racing on the same slot in the same millisecond, or + * from keys evicting each other. + * + *

Run: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=EvaluationMetadataTableBenchmark + * -Pjmh.profilers=gc}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 2, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Threads(12) +@Fork(value = 1) +public class EvaluationMetadataTableBenchmark { + private static final String CONFIG_PATH = + "dd-openfeature/src/test/fixtures/ffe-system-test-data/evaluation-suites/scale/ufc-config.json"; + private static final int CONTEXTS = 1024; + + /** The number of different flags each request evaluates. */ + @Param({"10", "50"}) + public int flagsPerRequest; + + /** The requested table size, 0 for no table. */ + @Param({"0", "256", "1024", "4096"}) + public int tableSize; + + private ServerConfiguration config; + private FlagEvaluator evaluator; + private String[] flags; + private EvaluationContext[] contexts; + private final LongAdder misses = new LongAdder(); + private final LongAdder lookups = new LongAdder(); + private long iterationStart; + + /** The position of one benchmark thread in the contexts. */ + @State(Scope.Thread) + public static class Requests { + private int next; + + EvaluationContext next(final EvaluationContext[] contexts) { + final EvaluationContext context = contexts[next]; + next = (next + 1) & (contexts.length - 1); + return context; + } + } + + @Setup(Level.Trial) + public void setUp() throws IOException { + config = UniversalFlagConfigParser.parse(Files.readAllBytes(configFile())); + evaluator = + new FlagEvaluator( + false, + (flag, subject, allocation, variant, serialId) -> false, + ExperimentationOptOutMode.DISABLED, + EvaluationMetadataTable.create(tableSize, misses)); + // Every 10th bulk flag is disabled: skip them, so each evaluation resolves a split. + flags = new String[flagsPerRequest]; + for (int i = 0, flag = 0; i < flagsPerRequest; flag++) { + if (flag % 10 != 9) { + flags[i++] = String.format(Locale.ROOT, "scale-bulk-%05d", flag); + } + } + contexts = new EvaluationContext[CONTEXTS]; + for (int i = 0; i < CONTEXTS; i++) { + contexts[i] = + new ImmutableContext( + "subject-" + i, + Collections.singletonMap( + "cohort", new Value(String.format(Locale.ROOT, "cohort-%02d", i % 100)))); + } + for (final String flag : flags) { + final ProviderEvaluation result = + evaluator.evaluate(config, Boolean.class, flag, false, contexts[0]); + if (result.getErrorCode() != null || result.getVariant() == null) { + throw new IllegalStateException(flag + " did not resolve a split: " + result.getReason()); + } + } + } + + @Setup(Level.Iteration) + public void startIteration() { + misses.reset(); + lookups.reset(); + iterationStart = System.nanoTime(); + } + + @TearDown(Level.Iteration) + @SuppressForbidden + public void reportIteration() { + final double elapsedMs = (System.nanoTime() - iterationStart) / 1e6; + final long total = lookups.sum(); + final long missed = tableSize == 0 ? total : misses.sum(); + System.out.printf( + Locale.ROOT, + "%n[table %d, %d flags] lookups %d, misses %d, hit rate %.4f%%, misses/ms %.1f%n", + EvaluationMetadataTable.size(tableSize), + flagsPerRequest, + total, + missed, + total == 0 ? 0 : 100.0 * (total - missed) / total, + missed / elapsedMs); + } + + @Benchmark + public void request(final Requests requests, final Blackhole blackhole) { + final EvaluationContext context = requests.next(contexts); + for (final String flag : flags) { + blackhole.consume(evaluator.evaluate(config, Boolean.class, flag, false, context)); + } + lookups.add(flags.length); + } + + private static Path configFile() { + Path directory = Paths.get("").toAbsolutePath(); + while (directory != null) { + final Path candidate = directory.resolve(CONFIG_PATH); + if (Files.exists(candidate)) { + return candidate; + } + directory = directory.getParent(); + } + throw new IllegalStateException( + "Unable to find the FFE scale suite: initialize the ffe-system-test-data submodule"); + } +} diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/DDEvaluator.java b/dd-openfeature/src/main/java/com/datadog/openfeature/DDEvaluator.java index fe32150a5b5..dc958ed11c8 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/DDEvaluator.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/DDEvaluator.java @@ -111,7 +111,10 @@ class DDEvaluator implements Evaluator, Consumer { // evaluations of allocations logging exposures, for the exposure hook. this.core = new FlagEvaluator( - settings.isSpanEnrichmentEnabled(), this::isExposed, experimentationOptOutMode); + settings.isSpanEnrichmentEnabled(), + this::isExposed, + experimentationOptOutMode, + settings.evaluationMetadataTableSize()); } @Override diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java index d36df1462ee..d0d2499b82f 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java @@ -39,6 +39,8 @@ public final class Settings { static final String EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED = "feature.flags.evaluation.telemetry.per.subject.opt.out.enabled"; static final String EXPERIMENTATION_OPT_OUT_MODE = "feature.flags.experimentation.opt.out.mode"; + static final String EVALUATION_METADATA_TABLE_SIZE = + "feature.flags.evaluation.metadata.table.size"; static final String TRACE_DEBUG = "trace.debug"; static final String API_KEY = "api-key"; static final String SITE = "site"; @@ -49,6 +51,7 @@ public final class Settings { static final String DEFAULT_SITE = "datadoghq.com"; static final int DEFAULT_POLL_INTERVAL_SECONDS = 30; static final int DEFAULT_REQUEST_TIMEOUT_SECONDS = 5; + static final int DEFAULT_EVALUATION_METADATA_TABLE_SIZE = 1024; private final AgentBridge bridge; private final UnaryOperator systemProperties; @@ -182,6 +185,17 @@ public String experimentationOptOutMode() { return mode == null ? "disabled" : mode; } + /** + * The number of slots of the table sharing the flag metadata of evaluations that resolve the same + * split in the same millisecond. The evaluator rounds it up to a power of two. + * + * @return the number of slots, or 0 to build the metadata of each evaluation (1024 by default). + */ + public int evaluationMetadataTableSize() { + return getNonNegativeInt( + EVALUATION_METADATA_TABLE_SIZE, DEFAULT_EVALUATION_METADATA_TABLE_SIZE); + } + /** * Tells whether {@code DD_TRACE_DEBUG} is on. It gates the diagnostics that log raw evaluation * contexts and values, on top of the application's logging configuration. @@ -292,6 +306,16 @@ private Boolean getBoolean(final String key) { /** Reads the first value that parses, in source priority order, like the agent. */ private int getPositiveInt(final String key, final int defaultValue) { + return getInt(key, defaultValue, 1, "positive"); + } + + /** Reads the first value that parses, in source priority order, like the agent. */ + private int getNonNegativeInt(final String key, final int defaultValue) { + return getInt(key, defaultValue, 0, "zero or positive"); + } + + private int getInt( + final String key, final int defaultValue, final int minimum, final String expected) { for (int source = 0; source < SOURCE_COUNT; source++) { final String value = value(key, source); if (nonBlank(value) == null) { @@ -299,10 +323,10 @@ private int getPositiveInt(final String key, final int defaultValue) { } try { final int parsed = Integer.parseInt(value); - if (parsed > 0) { + if (parsed >= minimum) { return parsed; } - LOGGER.warn("Invalid value for {}: {}. The value must be positive", key, value); + LOGGER.warn("Invalid value for {}: {}. The value must be {}", key, value, expected); return defaultValue; } catch (final NumberFormatException e) { LOGGER.warn("Invalid value for {}: {}", key, value); diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java new file mode 100644 index 00000000000..432d456e455 --- /dev/null +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java @@ -0,0 +1,182 @@ +package com.datadog.openfeature.internal.evaluation; + +import com.datadog.openfeature.internal.ufc.Allocation; +import com.datadog.openfeature.internal.ufc.Flag; +import com.datadog.openfeature.internal.ufc.Split; +import dev.openfeature.sdk.ImmutableMetadata; +import java.util.concurrent.atomic.LongAdder; +import javax.annotation.Nullable; + +/** + * Shares the flag metadata of successful evaluations that resolve the same split in the same + * millisecond. This is a burst coalescer, not a cache: an entry is only reusable within the + * millisecond it was built in, because the metadata carries the evaluation timestamp at millisecond + * resolution. A later millisecond builds and publishes a new entry. + * + *

The table is a fixed, direct-mapped array of slots. Each slot holds the metadata built last + * for one key: the {@link Split}, {@link Allocation} and {@link Flag} references, the configuration + * consent to observe full evaluation data, the exposure state, and the evaluation millisecond. + * Everything else in the metadata is fixed by the split, its allocation and its flag. The key + * compares references, so a reloaded configuration, whose objects are new, never matches an entry + * of the previous one, and no metadata field can be left out of the key. + * + *

The table takes no lock. An entry has only final fields and is published with a single + * reference store, so a reader sees either a whole entry or none. Two evaluations that miss the + * same slot at once both build metadata and the last store wins: a lost race only costs a rebuild. + */ +final class EvaluationMetadataTable { + /** The default number of slots. */ + static final int DEFAULT_SIZE = 1024; + + /** The largest number of slots. */ + static final int MAX_SIZE = 1 << 16; + + /** The evaluation logs no exposure, so the metadata carries no exposure state. */ + static final int NO_EXPOSURE = 0; + + /** The evaluation logs an exposure that was not recorded yet. */ + static final int NEW_EXPOSURE = 1; + + /** The evaluation logs an exposure that was already recorded. */ + static final int RECORDED_EXPOSURE = 2; + + private final Entry[] slots; + private final int mask; + @Nullable private final LongAdder misses; + + private EvaluationMetadataTable(final int size, @Nullable final LongAdder misses) { + this.slots = new Entry[size]; + this.mask = size - 1; + this.misses = misses; + } + + /** + * Creates a table. + * + * @param requestedSize the requested number of slots: rounded up to a power of two and capped at + * {@link #MAX_SIZE}, or 0 to share no metadata. + * @return the table, or {@code null} if the requested size is 0 or negative. + */ + @Nullable + static EvaluationMetadataTable create(final int requestedSize) { + return create(requestedSize, null); + } + + /** + * Creates a table that counts its misses, for the benchmarks. + * + * @param requestedSize the requested number of slots, as for {@link #create(int)}. + * @param misses the counter incremented on each lookup that finds no entry, or {@code null}. + * @return the table, or {@code null} if the requested size is 0 or negative. + */ + @Nullable + static EvaluationMetadataTable create(final int requestedSize, @Nullable final LongAdder misses) { + final int size = size(requestedSize); + return size == 0 ? null : new EvaluationMetadataTable(size, misses); + } + + /** + * @param requestedSize the requested number of slots. + * @return the number of slots of a table of this requested size, 0 when it shares no metadata. + */ + static int size(final int requestedSize) { + if (requestedSize <= 0) { + return 0; + } + if (requestedSize >= MAX_SIZE) { + return MAX_SIZE; + } + return requestedSize == 1 ? 1 : Integer.highestOneBit(requestedSize - 1) << 1; + } + + /** The number of slots. */ + int size() { + return slots.length; + } + + /** + * Finds the metadata built for this key. + * + * @return the metadata, or {@code null} if the slot holds another key. + */ + @Nullable + ImmutableMetadata get( + final Flag flag, + final Allocation allocation, + final Split split, + final boolean observeFullEvaluationData, + final int exposure, + final long evalTimestampMs) { + final Entry entry = slots[index(split, exposure)]; + if (entry != null + && entry.split == split + && entry.evalTimestampMs == evalTimestampMs + && entry.exposure == exposure + && entry.observeFullEvaluationData == observeFullEvaluationData + && entry.allocation == allocation + && entry.flag == flag) { + return entry.metadata; + } + if (misses != null) { + misses.increment(); + } + return null; + } + + /** Publishes the metadata built for this key, replacing the slot's entry. */ + void put( + final Flag flag, + final Allocation allocation, + final Split split, + final boolean observeFullEvaluationData, + final int exposure, + final long evalTimestampMs, + final ImmutableMetadata metadata) { + slots[index(split, exposure)] = + new Entry( + flag, + allocation, + split, + observeFullEvaluationData, + exposure, + evalTimestampMs, + metadata); + } + + private int index(final Split split, final int exposure) { + // The exposure states of a split take neighboring slots, so a subject that was already exposed + // does not evict the entry of one that was not. The millisecond is left out: the next one + // replaces the entry of the previous one instead of evicting another split. + int hash = (System.identityHashCode(split) << 2 | exposure) * 0x9E3779B9; + hash ^= hash >>> 16; + return hash & mask; + } + + /** One immutable slot entry. Its final fields make it safe to publish without a barrier. */ + private static final class Entry { + final Flag flag; + final Allocation allocation; + final Split split; + final boolean observeFullEvaluationData; + final int exposure; + final long evalTimestampMs; + final ImmutableMetadata metadata; + + Entry( + final Flag flag, + final Allocation allocation, + final Split split, + final boolean observeFullEvaluationData, + final int exposure, + final long evalTimestampMs, + final ImmutableMetadata metadata) { + this.flag = flag; + this.allocation = allocation; + this.split = split; + this.observeFullEvaluationData = observeFullEvaluationData; + this.exposure = exposure; + this.evalTimestampMs = evalTimestampMs; + this.metadata = metadata; + } + } +} diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/FlagEvaluator.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/FlagEvaluator.java index 55ba7dc45ce..2a5bfb18f8a 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/FlagEvaluator.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/FlagEvaluator.java @@ -43,6 +43,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +62,7 @@ public final class FlagEvaluator { private final boolean spanEnrichmentEnabled; private final ExposureLookup exposureLookup; private final ExperimentationOptOutMode experimentationOptOutMode; + @Nullable private final EvaluationMetadataTable metadataTable; /** * Creates an evaluator. @@ -76,9 +78,44 @@ public FlagEvaluator( final boolean spanEnrichmentEnabled, final ExposureLookup exposureLookup, final ExperimentationOptOutMode experimentationOptOutMode) { + this( + spanEnrichmentEnabled, + exposureLookup, + experimentationOptOutMode, + EvaluationMetadataTable.DEFAULT_SIZE); + } + + /** + * Creates an evaluator. + * + * @param spanEnrichmentEnabled whether span enrichment is enabled. + * @param exposureLookup the lookup telling whether an exposure was already recorded. + * @param experimentationOptOutMode the experimentation opt-out mode. + * @param metadataTableSize the number of slots of the table sharing the metadata of evaluations + * that resolve the same split in the same millisecond, rounded up to a power of two, or 0 to + * build the metadata of each evaluation. + */ + public FlagEvaluator( + final boolean spanEnrichmentEnabled, + final ExposureLookup exposureLookup, + final ExperimentationOptOutMode experimentationOptOutMode, + final int metadataTableSize) { + this( + spanEnrichmentEnabled, + exposureLookup, + experimentationOptOutMode, + EvaluationMetadataTable.create(metadataTableSize)); + } + + FlagEvaluator( + final boolean spanEnrichmentEnabled, + final ExposureLookup exposureLookup, + final ExperimentationOptOutMode experimentationOptOutMode, + @Nullable final EvaluationMetadataTable metadataTable) { this.spanEnrichmentEnabled = spanEnrichmentEnabled; this.exposureLookup = exposureLookup; this.experimentationOptOutMode = experimentationOptOutMode; + this.metadataTable = metadataTable; } public ProviderEvaluation evaluate( @@ -490,6 +527,56 @@ private ProviderEvaluation resolveVariant( observeFullEvaluationData); } + // The exposure state depends on the subject, so it is part of the metadata table key. + final boolean doLog = Boolean.TRUE.equals(allocation.doLog); + final int exposure; + if (!doLog) { + exposure = EvaluationMetadataTable.NO_EXPOSURE; + } else if (this.exposureLookup.isExposed( + flag.key, context.getTargetingKey(), allocation.key, variant.key, split.serialId)) { + exposure = EvaluationMetadataTable.RECORDED_EXPOSURE; + } else { + exposure = EvaluationMetadataTable.NEW_EXPOSURE; + } + // Everything else in the metadata is fixed by the split, so evaluations of the split in the + // same millisecond share it. + final EvaluationMetadataTable table = this.metadataTable; + ImmutableMetadata metadata = + table == null + ? null + : table.get( + flag, allocation, split, observeFullEvaluationData, exposure, evalTimestampMs); + if (metadata == null) { + metadata = + buildMetadata( + flag, allocation, split, observeFullEvaluationData, doLog, exposure, evalTimestampMs); + if (table != null) { + table.put( + flag, + allocation, + split, + observeFullEvaluationData, + exposure, + evalTimestampMs, + metadata); + } + } + return ProviderEvaluation.builder() + .value(mappedValue) + .reason(resolveReason(allocation, split)) + .variant(variant.key) + .flagMetadata(metadata) + .build(); + } + + private ImmutableMetadata buildMetadata( + final Flag flag, + final Allocation allocation, + final Split split, + final boolean observeFullEvaluationData, + final boolean doLog, + final int exposure, + final long evalTimestampMs) { // Stamp eval-time at the resolution point so first/last_evaluation reflect evaluation time, // not hook-fire time. Passed to the hook via provider metadata "__dd_eval_timestamp_ms". final ImmutableMetadata.ImmutableMetadataBuilder metadataBuilder = @@ -503,7 +590,6 @@ private ProviderEvaluation resolveVariant( // for the exposure hook, which sends the exposure after the evaluation succeeds. An evaluation // that neither logs exposures nor enriches spans carries neither key. // __dd_split_serial_id is omitted when the split carries no serial id. - final boolean doLog = Boolean.TRUE.equals(allocation.doLog); if (spanEnrichmentEnabled || doLog) { if (split.serialId != null) { metadataBuilder.addInteger(SPLIT_SERIAL_ID, split.serialId); @@ -513,16 +599,9 @@ private ProviderEvaluation resolveVariant( addSplitFeatures(metadataBuilder, split); if (doLog) { metadataBuilder.addBoolean( - EXPOSURE_CACHE_HIT, - this.exposureLookup.isExposed( - flag.key, context.getTargetingKey(), allocation.key, variant.key, split.serialId)); + EXPOSURE_CACHE_HIT, exposure == EvaluationMetadataTable.RECORDED_EXPOSURE); } - return ProviderEvaluation.builder() - .value(mappedValue) - .reason(resolveReason(allocation, split)) - .variant(variant.key) - .flagMetadata(metadataBuilder.build()) - .build(); + return metadataBuilder.build(); } private static String resolveReason(final Allocation allocation, final Split split) { diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java index 6fc2d6c6d0f..d6853ef5798 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java @@ -90,6 +90,7 @@ void appliesDefaults() { assertTrue(settings.isEvaluationCountsEnabled()); assertTrue(settings.isExposuresDatadogLoggingEnabled()); assertEquals("disabled", settings.experimentationOptOutMode()); + assertEquals(1024, settings.evaluationMetadataTableSize()); } @TableTest({ @@ -180,6 +181,21 @@ void fallsBackToDefaultPollIntervalOnInvalidValues(final String value, final int .agentlessPollIntervalSeconds()); } + @TableTest({ + "scenario | value | size", + "disabled | 0 | 0 ", + "custom | 300 | 300 ", + "negative | -1 | 1024", + "invalid | ten | 1024", + "blank | ' ' | 1024" + }) + void readsTheEvaluationMetadataTableSize(final String value, final int size) { + assertEquals( + size, + TestSettings.of("feature.flags.evaluation.metadata.table.size", value) + .evaluationMetadataTableSize()); + } + @Test void fallsBackToLowerPrioritySourcesOnBlankOrInvalidValues() { final Map properties = new HashMap<>(); diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java new file mode 100644 index 00000000000..28cbf5ca8c7 --- /dev/null +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java @@ -0,0 +1,294 @@ +package com.datadog.openfeature.internal.evaluation; + +import static com.datadog.openfeature.internal.evaluation.EvaluationMetadataTable.NEW_EXPOSURE; +import static com.datadog.openfeature.internal.evaluation.EvaluationMetadataTable.RECORDED_EXPOSURE; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.DO_LOG; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVAL_TIMESTAMP_MS; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_CACHE_HIT; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.OBSERVE_FULL_EVALUATION_DATA; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datadog.openfeature.ExperimentationOptOutMode; +import com.datadog.openfeature.internal.ufc.Allocation; +import com.datadog.openfeature.internal.ufc.Flag; +import com.datadog.openfeature.internal.ufc.ServerConfiguration; +import com.datadog.openfeature.internal.ufc.Split; +import com.datadog.openfeature.internal.ufc.ValueType; +import com.datadog.openfeature.internal.ufc.Variant; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.MutableContext; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class EvaluationMetadataTableTest { + private static final ImmutableMetadata METADATA = ImmutableMetadata.builder().build(); + + @TableTest({ + "scenario | requested | size ", + "disabled | 0 | 0 ", + "negative | -1 | 0 ", + "one | 1 | 1 ", + "rounded up | 3 | 4 ", + "power of two | 1024 | 1024 ", + "next power | 1025 | 2048 ", + "capped | 100000 | 65536" + }) + void roundsTheSizeUpToAPowerOfTwo(final int requested, final int size) { + assertEquals(size, EvaluationMetadataTable.size(requested)); + final EvaluationMetadataTable table = EvaluationMetadataTable.create(requested); + if (size == 0) { + assertNull(table); + } else { + assertEquals(size, table.size()); + } + } + + @Test + void findsTheMetadataOfTheSameKeyInTheSameMillisecondOnly() { + final EvaluationMetadataTable table = EvaluationMetadataTable.create(16); + final Split split = split(); + final Allocation allocation = allocation("allocation", split, true); + final Flag flag = flag("flag", allocation); + table.put(flag, allocation, split, true, NEW_EXPOSURE, 100L, METADATA); + + assertSame(METADATA, table.get(flag, allocation, split, true, NEW_EXPOSURE, 100L)); + assertNull(table.get(flag, allocation, split, true, NEW_EXPOSURE, 101L)); + assertNull(table.get(flag, allocation, split, false, NEW_EXPOSURE, 100L)); + assertNull(table.get(flag, allocation, split, true, RECORDED_EXPOSURE, 100L)); + assertNull(table.get(flag, allocation, split(), true, NEW_EXPOSURE, 100L)); + final Allocation other = allocation("allocation", split, true); + assertNull(table.get(flag, other, split, true, NEW_EXPOSURE, 100L)); + assertNull(table.get(flag("flag", allocation), allocation, split, true, NEW_EXPOSURE, 100L)); + } + + @Test + void evaluationsInTheSameMillisecondShareTheMetadata() { + final FlagEvaluator evaluator = evaluator(EvaluationMetadataTable.DEFAULT_SIZE); + final ServerConfiguration config = config(true, flag("flag", allocation("a", split(), true))); + int shared = 0; + for (int i = 0; i < 1000 && shared < 10; i++) { + final ImmutableMetadata first = evaluate(evaluator, config, "flag", "user"); + final ImmutableMetadata second = evaluate(evaluator, config, "flag", "user"); + if (first.getLong(EVAL_TIMESTAMP_MS).equals(second.getLong(EVAL_TIMESTAMP_MS))) { + assertSame(first, second); + shared++; + } else { + assertNotSame(first, second); + } + } + assertEquals(10, shared); + } + + @Test + void aLaterMillisecondBuildsNewMetadata() { + final FlagEvaluator evaluator = evaluator(EvaluationMetadataTable.DEFAULT_SIZE); + final ServerConfiguration config = config(true, flag("flag", allocation("a", split(), true))); + final ImmutableMetadata first = evaluate(evaluator, config, "flag", "user"); + final long firstMs = first.getLong(EVAL_TIMESTAMP_MS); + ImmutableMetadata later; + do { + later = evaluate(evaluator, config, "flag", "user"); + } while (later.getLong(EVAL_TIMESTAMP_MS) == firstMs); + + assertNotSame(first, later); + assertTrue(later.getLong(EVAL_TIMESTAMP_MS) > firstMs); + final Map expected = new HashMap<>(first.asUnmodifiableMap()); + expected.put(EVAL_TIMESTAMP_MS, later.getLong(EVAL_TIMESTAMP_MS)); + assertEquals(expected, later.asUnmodifiableMap()); + } + + @Test + void keysOnTheConsentOfTheConfiguration() { + final FlagEvaluator evaluator = evaluator(EvaluationMetadataTable.DEFAULT_SIZE); + final Flag flag = flag("flag", allocation("a", split(), true)); + final ServerConfiguration consenting = config(true, flag); + final ServerConfiguration withheld = config(false, flag); + + for (int i = 0; i < 100; i++) { + assertEquals( + true, + evaluate(evaluator, consenting, "flag", "user").getBoolean(OBSERVE_FULL_EVALUATION_DATA)); + assertEquals( + false, + evaluate(evaluator, withheld, "flag", "user").getBoolean(OBSERVE_FULL_EVALUATION_DATA)); + } + } + + @Test + void keysOnTheExposureState() { + final FlagEvaluator evaluator = evaluator(EvaluationMetadataTable.DEFAULT_SIZE); + final ServerConfiguration config = config(true, flag("flag", allocation("a", split(), true))); + + for (int i = 0; i < 100; i++) { + assertEquals( + true, evaluate(evaluator, config, "flag", "exposed").getBoolean(EXPOSURE_CACHE_HIT)); + assertEquals( + false, evaluate(evaluator, config, "flag", "user").getBoolean(EXPOSURE_CACHE_HIT)); + } + } + + @Test + void keysOnTheAllocationAndItsDoLog() { + final FlagEvaluator evaluator = evaluator(EvaluationMetadataTable.DEFAULT_SIZE); + // Directly built configurations can share a split between allocations. + final Split split = split(); + final ServerConfiguration logging = config(true, flag("flag", allocation("a", split, true))); + final ServerConfiguration silent = config(true, flag("flag", allocation("b", split, false))); + + for (int i = 0; i < 100; i++) { + final ImmutableMetadata logged = evaluate(evaluator, logging, "flag", "user"); + final ImmutableMetadata unlogged = evaluate(evaluator, silent, "flag", "user"); + assertEquals("a", logged.getString(ALLOCATION_KEY)); + assertEquals(true, logged.getBoolean(DO_LOG)); + assertEquals("b", unlogged.getString(ALLOCATION_KEY)); + assertNull(unlogged.getBoolean(DO_LOG)); + assertNull(unlogged.getBoolean(EXPOSURE_CACHE_HIT)); + } + } + + @Test + void aReloadedConfigurationBuildsNewMetadata() { + final FlagEvaluator evaluator = evaluator(EvaluationMetadataTable.DEFAULT_SIZE); + final ServerConfiguration before = config(true, flag("flag", allocation("a", split(), true))); + final ServerConfiguration after = config(true, flag("flag", allocation("b", split(), true))); + + for (int i = 0; i < 100; i++) { + final ImmutableMetadata old = evaluate(evaluator, before, "flag", "user"); + final ImmutableMetadata reloaded = evaluate(evaluator, after, "flag", "user"); + assertNotSame(old, reloaded); + assertEquals("a", old.getString(ALLOCATION_KEY)); + assertEquals("b", reloaded.getString(ALLOCATION_KEY)); + } + } + + @Test + void aZeroSizeBuildsTheMetadataOfEachEvaluation() { + final FlagEvaluator evaluator = evaluator(0); + final ServerConfiguration config = config(true, flag("flag", allocation("a", split(), true))); + for (int i = 0; i < 100; i++) { + final ImmutableMetadata first = evaluate(evaluator, config, "flag", "user"); + final ImmutableMetadata second = evaluate(evaluator, config, "flag", "user"); + assertNotSame(first, second); + if (first.getLong(EVAL_TIMESTAMP_MS).equals(second.getLong(EVAL_TIMESTAMP_MS))) { + assertEquals(first.asUnmodifiableMap(), second.asUnmodifiableMap()); + } + } + } + + @Test + void neverReturnsTheMetadataOfAnotherKeyUnderContention() throws Exception { + // A single slot makes every key collide. + final FlagEvaluator evaluator = + new FlagEvaluator( + false, + exposedUser(), + ExperimentationOptOutMode.DISABLED, + EvaluationMetadataTable.create(1)); + final List flags = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + flags.add(flag("flag-" + i, allocation("allocation-" + i, split(), i % 2 == 0))); + } + final ServerConfiguration consenting = config(true, flags.toArray(new Flag[0])); + final ServerConfiguration withheld = config(false, flags.toArray(new Flag[0])); + final int threads = 8; + final CountDownLatch start = new CountDownLatch(1); + final ExecutorService executor = Executors.newFixedThreadPool(threads); + try { + final List> results = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + results.add( + executor.submit(() -> { + start.await(); + final ThreadLocalRandom random = ThreadLocalRandom.current(); + for (int i = 0; i < 50_000; i++) { + final int flag = random.nextInt(flags.size()); + final boolean consent = random.nextBoolean(); + final String subject = random.nextBoolean() ? "exposed" : "user"; + final long before = System.currentTimeMillis(); + final ImmutableMetadata metadata = + evaluate(evaluator, consent ? consenting : withheld, "flag-" + flag, subject); + final long after = System.currentTimeMillis(); + assertEquals("flag-" + flag, metadata.getString("flagKey")); + assertEquals("allocation-" + flag, metadata.getString(ALLOCATION_KEY)); + assertEquals(consent, metadata.getBoolean(OBSERVE_FULL_EVALUATION_DATA)); + assertEquals( + flag % 2 == 0 ? "exposed".equals(subject) : null, + metadata.getBoolean(EXPOSURE_CACHE_HIT)); + final long ms = metadata.getLong(EVAL_TIMESTAMP_MS); + assertTrue(before <= ms && ms <= after, ms + " not in " + before + ".." + after); + } + return null; + })); + } + start.countDown(); + for (final Future result : results) { + result.get(60, TimeUnit.SECONDS); + } + } finally { + executor.shutdownNow(); + } + } + + private static FlagEvaluator evaluator(final int tableSize) { + return new FlagEvaluator(false, exposedUser(), ExperimentationOptOutMode.DISABLED, tableSize); + } + + private static FlagEvaluator.ExposureLookup exposedUser() { + return (flag, subject, allocation, variant, serialId) -> "exposed".equals(subject); + } + + private static ImmutableMetadata evaluate( + final FlagEvaluator evaluator, + final ServerConfiguration config, + final String flag, + final String subject) { + final ImmutableMetadata metadata = + evaluator + .evaluate(config, Integer.class, flag, 0, new MutableContext(subject)) + .getFlagMetadata(); + assertNotNull(metadata.getLong(EVAL_TIMESTAMP_MS)); + return metadata; + } + + private static Split split() { + return new Split(emptyList(), "on", emptyMap(), 7); + } + + private static Allocation allocation(final String key, final Split split, final boolean doLog) { + return new Allocation(key, null, null, null, singletonList(split), doLog); + } + + private static Flag flag(final String key, final Allocation allocation) { + final Map variations = new HashMap<>(); + variations.put("on", new Variant("on", 1)); + return new Flag(key, true, ValueType.INTEGER, variations, singletonList(allocation)); + } + + private static ServerConfiguration config(final boolean consent, final Flag... flags) { + final Map byKey = new HashMap<>(); + for (final Flag flag : asList(flags)) { + byKey.put(flag.key, flag); + } + return new ServerConfiguration("", "", consent, null, byKey); + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index a14f4adbf5c..e52012c2ed3 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -36,5 +36,14 @@ public final class FeatureFlaggingConfig { public static final String FEATURE_FLAGS_EXPERIMENTATION_OPT_OUT_MODE = "feature.flags.experimentation.opt.out.mode"; + /** + * The number of slots of the table sharing the flag metadata of evaluations that resolve the same + * split in the same millisecond, rounded up to a power of two. {@code 0} builds the metadata of + * each evaluation. Default: {@code 1024}. Maps to {@code + * DD_FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE}. + */ + public static final String FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE = + "feature.flags.evaluation.metadata.table.size"; + private FeatureFlaggingConfig() {} } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 0a42dda0ec2..f23c71d6881 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1612,6 +1612,14 @@ "aliases": [] } ], + "DD_FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE": [ + { + "version": "A", + "type": "int", + "default": "1024", + "aliases": [] + } + ], "DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED": [ { "version": "A", From 136f3eb5a24af0129c6179a24759128b21f16569 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 19 Nov 2026 03:32:32 +0000 Subject: [PATCH 17/18] Make the time bucket of the shared flag metadata configurable DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS sets the width of the time bucket in which evaluations of a split share their metadata. The default, 1 ms, keeps __dd_eval_timestamp_ms exact. A width of W ms lets an evaluation reuse metadata built earlier in its bucket, so the flag evaluation events report it up to W - 1 ms early. 0 leaves the timestamp out of the metadata, so the flag evaluation hook stamps its own time and a split's metadata is shared until its slot is replaced. Widths are capped at the 10 s flush interval of the evaluation events. Exposures stamp their own time and are not affected. The table benchmark now sweeps the table size and bucket width, with uniform and Zipf-skewed flag popularity, and reports hit rate and how early the reported timestamps are. Co-authored-by: Douglas Q Hawkins Co-Authored-By: Claude Opus 5.5 --- dd-openfeature/README.md | 1 + .../EvaluationMetadataTableBenchmark.java | 173 ++++++++++++++---- .../com/datadog/openfeature/DDEvaluator.java | 3 +- .../openfeature/internal/config/Settings.java | 17 ++ .../evaluation/EvaluationMetadataTable.java | 66 ++++--- .../internal/evaluation/FlagEvaluator.java | 56 ++++-- .../internal/config/SettingsTest.java | 15 ++ .../EvaluationMetadataTableTest.java | 103 ++++++++++- .../api/config/FeatureFlaggingConfig.java | 10 + metadata/supported-configurations.json | 8 + 10 files changed, 365 insertions(+), 87 deletions(-) diff --git a/dd-openfeature/README.md b/dd-openfeature/README.md index 25e617ed640..00d260aef11 100644 --- a/dd-openfeature/README.md +++ b/dd-openfeature/README.md @@ -55,6 +55,7 @@ Settings are read from the Datadog Java agent configuration when attached, then | `DD_FEATURE_FLAGS_EXPERIMENTATION_OPT_OUT_MODE` | `disabled` | Controls per-subject experimentation opt-out: `disabled`, `immediate`, or `future_experiments_only`. See [Per-subject experimentation opt-out](#per-subject-experimentation-opt-out). | | `DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED` | `false` | Lets the `dd_evaluation_telemetry_opt_out` context attribute opt subjects out of evaluation telemetry. See [Per-subject evaluation telemetry opt-out](#per-subject-evaluation-telemetry-opt-out). | | `DD_FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE` | `1024` | Slots of the table that lets evaluations resolving the same split in the same millisecond share their flag metadata, rounded up to a power of two (at most 65536). `0` builds the metadata of every evaluation. | +| `DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS` | `1` | Width, in milliseconds, of the time bucket in which evaluations of the same split share their flag metadata (at most 10000). The `first_evaluation` and `last_evaluation` times of flag evaluation events can then be up to the width minus one millisecond early; `1` keeps them exact. `0` leaves the time out of the shared metadata, and the evaluation events use the time their hook runs. | | `DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED` | `false` | Adds flag evaluations to the local root span. Requires `dd-java-agent`. | | `DD_API_KEY`, `DD_SITE`, `DD_ENV`, `DD_SERVICE`, `DD_VERSION` | | Credentials and event context. | | `DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED` | | Deprecated: `true` selects `remote_config`, `false` disables Feature Flags. An explicit source wins. | diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableBenchmark.java index 83144bae84d..895613c9c19 100644 --- a/dd-openfeature/src/jmh/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableBenchmark.java +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableBenchmark.java @@ -1,5 +1,6 @@ package com.datadog.openfeature.internal.evaluation; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVAL_TIMESTAMP_MS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -15,8 +16,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Locale; +import java.util.SplittableRandom; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.LongAdder; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -30,41 +36,43 @@ import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; /** - * Concurrent requests that each evaluate the same {@code flagsPerRequest} flags of the 10,000-flag - * scale configuration ({@code evaluation-suites/scale}), for a subject and cohort that change on - * every request, on 12 threads sharing one evaluator. It shows what the evaluation metadata table - * saves when many threads resolve the same splits in the same millisecond, and whether its slot - * writes contend. {@code tableSize=0} builds the metadata of every evaluation, as without the - * table. + * Requests that each evaluate {@code flagsPerRequest} flags drawn from 1,000 enabled flags of the + * 10,000-flag scale configuration ({@code evaluation-suites/scale}), for a subject and cohort that + * change on every request, on threads sharing one evaluator. It sweeps the evaluation metadata + * table size and time bucket width. {@code tableSize=0} builds the metadata of every evaluation, as + * without the table; {@code timeBucketMs=0} leaves the timestamp out of the metadata. * - *

An operation is one request. Each measurement iteration prints the table's hit rate (the share - * of successful evaluations that reused metadata) and its misses per millisecond. The lowest - * possible miss rate is one per distinct key per millisecond: a request resolves {@code - * flagsPerRequest} splits, and a cohort can switch one of them, so about {@code flagsPerRequest + - * 1} keys. Misses above that come from threads racing on the same slot in the same millisecond, or - * from keys evicting each other. + *

{@code popularity=uniform} draws each flag of a request uniformly; {@code skewed} draws it + * from a Zipf distribution with exponent 1, so a few flags take most evaluations. An operation is + * one request, sampled for its mean and percentiles. Each measurement iteration prints the table's + * hit rate, the share of successful evaluations that reused metadata, and how early the reported + * {@code __dd_eval_timestamp_ms} was: the clock just before the evaluation minus the reported time, + * sampled on every 8th request. Without a timestamp in the metadata, the flag evaluation hook reads + * its own clock after the evaluation, so the reported time is never early. * - *

Run: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=EvaluationMetadataTableBenchmark - * -Pjmh.profilers=gc}. + *

Run with the thread count as an option: {@code java -jar dd-openfeature/build/libs/*-jmh.jar + * EvaluationMetadataTableBenchmark -t 12 -prof gc}. {@code tableSize=0} ignores the bucket width. */ @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) -@Measurement(iterations = 5, time = 2, timeUnit = SECONDS) -@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 2, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = SECONDS) +@BenchmarkMode(Mode.SampleTime) @OutputTimeUnit(NANOSECONDS) -@Threads(12) @Fork(value = 1) public class EvaluationMetadataTableBenchmark { private static final String CONFIG_PATH = "dd-openfeature/src/test/fixtures/ffe-system-test-data/evaluation-suites/scale/ufc-config.json"; + private static final int POPULATION = 1000; private static final int CONTEXTS = 1024; + private static final int DRAWS = 1 << 16; + private static final int ERROR_SAMPLE_MASK = 7; + private static final int MAX_ERROR_MS = 10_000; - /** The number of different flags each request evaluates. */ + /** The number of flags each request evaluates. */ @Param({"10", "50"}) public int flagsPerRequest; @@ -72,22 +80,56 @@ public class EvaluationMetadataTableBenchmark { @Param({"0", "256", "1024", "4096"}) public int tableSize; + /** The time bucket width in milliseconds, 0 to leave the timestamp out of the metadata. */ + @Param({"1", "10", "100", "1000", "0"}) + public int timeBucketMs; + + /** How the flags of a request are drawn: {@code uniform} or {@code skewed} (Zipf). */ + @Param({"uniform", "skewed"}) + public String popularity; + private ServerConfiguration config; private FlagEvaluator evaluator; private String[] flags; + private double[] cumulative; private EvaluationContext[] contexts; + private final AtomicInteger threadSeeds = new AtomicInteger(); + private final List threads = Collections.synchronizedList(new ArrayList<>()); private final LongAdder misses = new LongAdder(); private final LongAdder lookups = new LongAdder(); - private long iterationStart; - /** The position of one benchmark thread in the contexts. */ + /** One benchmark thread: its flag draws, its positions, and its timestamp error histogram. */ @State(Scope.Thread) public static class Requests { - private int next; + private int[] draws; + private int nextDraw; + private int nextContext; + private int requests; + private long[] earlyMs; + + @Setup(Level.Trial) + public void setUp(final EvaluationMetadataTableBenchmark benchmark) { + final int seed = benchmark.threadSeeds.incrementAndGet(); + draws = benchmark.draws(new SplittableRandom(seed)); + nextContext = (seed * 97) & (CONTEXTS - 1); + earlyMs = new long[MAX_ERROR_MS + 1]; + benchmark.threads.add(this); + } + + @Setup(Level.Iteration) + public void reset() { + Arrays.fill(earlyMs, 0); + } + + int nextDraw() { + final int draw = draws[nextDraw]; + nextDraw = (nextDraw + 1) & (DRAWS - 1); + return draw; + } - EvaluationContext next(final EvaluationContext[] contexts) { - final EvaluationContext context = contexts[next]; - next = (next + 1) & (contexts.length - 1); + EvaluationContext nextContext(final EvaluationContext[] contexts) { + final EvaluationContext context = contexts[nextContext]; + nextContext = (nextContext + 1) & (CONTEXTS - 1); return context; } } @@ -100,14 +142,24 @@ public void setUp() throws IOException { false, (flag, subject, allocation, variant, serialId) -> false, ExperimentationOptOutMode.DISABLED, - EvaluationMetadataTable.create(tableSize, misses)); + EvaluationMetadataTable.create(tableSize, misses), + timeBucketMs); // Every 10th bulk flag is disabled: skip them, so each evaluation resolves a split. - flags = new String[flagsPerRequest]; - for (int i = 0, flag = 0; i < flagsPerRequest; flag++) { + flags = new String[POPULATION]; + for (int i = 0, flag = 0; i < POPULATION; flag++) { if (flag % 10 != 9) { flags[i++] = String.format(Locale.ROOT, "scale-bulk-%05d", flag); } } + cumulative = new double[POPULATION]; + double sum = 0; + for (int i = 0; i < POPULATION; i++) { + sum += "skewed".equals(popularity) ? 1.0 / (i + 1) : 1.0; + cumulative[i] = sum; + } + for (int i = 0; i < POPULATION; i++) { + cumulative[i] /= sum; + } contexts = new EvaluationContext[CONTEXTS]; for (int i = 0; i < CONTEXTS; i++) { contexts[i] = @@ -125,37 +177,82 @@ public void setUp() throws IOException { } } + int[] draws(final SplittableRandom random) { + final int[] draws = new int[DRAWS]; + for (int i = 0; i < DRAWS; i++) { + final int index = Arrays.binarySearch(cumulative, random.nextDouble()); + draws[i] = Math.min(index < 0 ? -index - 1 : index, POPULATION - 1); + } + return draws; + } + @Setup(Level.Iteration) public void startIteration() { misses.reset(); lookups.reset(); - iterationStart = System.nanoTime(); } @TearDown(Level.Iteration) @SuppressForbidden public void reportIteration() { - final double elapsedMs = (System.nanoTime() - iterationStart) / 1e6; final long total = lookups.sum(); final long missed = tableSize == 0 ? total : misses.sum(); + final long[] early = new long[MAX_ERROR_MS + 1]; + synchronized (threads) { + for (final Requests thread : threads) { + for (int i = 0; i <= MAX_ERROR_MS; i++) { + early[i] += thread.earlyMs[i]; + } + } + } + long samples = 0; + int max = 0; + for (int i = 0; i <= MAX_ERROR_MS; i++) { + samples += early[i]; + if (early[i] != 0) { + max = i; + } + } + int p99 = 0; + for (long seen = early[0]; seen < samples * 0.99 && p99 < MAX_ERROR_MS; ) { + seen += early[++p99]; + } System.out.printf( Locale.ROOT, - "%n[table %d, %d flags] lookups %d, misses %d, hit rate %.4f%%, misses/ms %.1f%n", + "%n@@ table=%d bucket=%d flags=%d popularity=%s lookups=%d misses=%d hitRate=%.4f" + + " earlyP99Ms=%d earlyMaxMs=%d samples=%d%n", EvaluationMetadataTable.size(tableSize), + timeBucketMs, flagsPerRequest, + popularity, total, missed, - total == 0 ? 0 : 100.0 * (total - missed) / total, - missed / elapsedMs); + total == 0 ? 0 : (double) (total - missed) / total, + p99, + max, + samples); } @Benchmark public void request(final Requests requests, final Blackhole blackhole) { - final EvaluationContext context = requests.next(contexts); - for (final String flag : flags) { - blackhole.consume(evaluator.evaluate(config, Boolean.class, flag, false, context)); + final EvaluationContext context = requests.nextContext(contexts); + final boolean sample = (requests.requests++ & ERROR_SAMPLE_MASK) == 0; + for (int i = 0; i < flagsPerRequest; i++) { + final String flag = flags[requests.nextDraw()]; + if (sample) { + final long before = System.currentTimeMillis(); + final ProviderEvaluation result = + evaluator.evaluate(config, Boolean.class, flag, false, context); + final Long reported = result.getFlagMetadata().getLong(EVAL_TIMESTAMP_MS); + // Without the timestamp, the hook reads its clock after the evaluation: never early. + final long early = reported == null ? 0 : before - reported; + requests.earlyMs[(int) Math.max(0, Math.min(early, MAX_ERROR_MS))]++; + blackhole.consume(result); + } else { + blackhole.consume(evaluator.evaluate(config, Boolean.class, flag, false, context)); + } } - lookups.add(flags.length); + lookups.add(flagsPerRequest); } private static Path configFile() { diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/DDEvaluator.java b/dd-openfeature/src/main/java/com/datadog/openfeature/DDEvaluator.java index dc958ed11c8..f35fd0574f5 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/DDEvaluator.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/DDEvaluator.java @@ -114,7 +114,8 @@ class DDEvaluator implements Evaluator, Consumer { settings.isSpanEnrichmentEnabled(), this::isExposed, experimentationOptOutMode, - settings.evaluationMetadataTableSize()); + settings.evaluationMetadataTableSize(), + settings.evaluationMetadataTimeBucketMs()); } @Override diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java index d0d2499b82f..854821d299c 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java @@ -41,6 +41,8 @@ public final class Settings { static final String EXPERIMENTATION_OPT_OUT_MODE = "feature.flags.experimentation.opt.out.mode"; static final String EVALUATION_METADATA_TABLE_SIZE = "feature.flags.evaluation.metadata.table.size"; + static final String EVALUATION_METADATA_TIME_BUCKET_MS = + "feature.flags.evaluation.metadata.time.bucket.ms"; static final String TRACE_DEBUG = "trace.debug"; static final String API_KEY = "api-key"; static final String SITE = "site"; @@ -52,6 +54,7 @@ public final class Settings { static final int DEFAULT_POLL_INTERVAL_SECONDS = 30; static final int DEFAULT_REQUEST_TIMEOUT_SECONDS = 5; static final int DEFAULT_EVALUATION_METADATA_TABLE_SIZE = 1024; + static final int DEFAULT_EVALUATION_METADATA_TIME_BUCKET_MS = 1; private final AgentBridge bridge; private final UnaryOperator systemProperties; @@ -196,6 +199,20 @@ public int evaluationMetadataTableSize() { EVALUATION_METADATA_TABLE_SIZE, DEFAULT_EVALUATION_METADATA_TABLE_SIZE); } + /** + * The width of the time bucket in which evaluations of a split share their flag metadata, + * including its evaluation timestamp. Evaluations that reuse the metadata report the time of the + * evaluation that built it, up to the width minus one millisecond early. The evaluator caps it at + * 10,000 milliseconds, the flush interval of the flag evaluation events. + * + * @return the width in milliseconds (1 by default, which keeps the timestamps exact), or 0 to + * leave the timestamp out of the metadata, so the hooks read their own clock. + */ + public int evaluationMetadataTimeBucketMs() { + return getNonNegativeInt( + EVALUATION_METADATA_TIME_BUCKET_MS, DEFAULT_EVALUATION_METADATA_TIME_BUCKET_MS); + } + /** * Tells whether {@code DD_TRACE_DEBUG} is on. It gates the diagnostics that log raw evaluation * contexts and values, on top of the application's logging configuration. diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java index 432d456e455..a9da7fc5d7e 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java @@ -8,17 +8,21 @@ import javax.annotation.Nullable; /** - * Shares the flag metadata of successful evaluations that resolve the same split in the same - * millisecond. This is a burst coalescer, not a cache: an entry is only reusable within the - * millisecond it was built in, because the metadata carries the evaluation timestamp at millisecond - * resolution. A later millisecond builds and publishes a new entry. + * Shares the flag metadata of successful evaluations that resolve the same split in the same time + * bucket. With a bounded bucket, this is a burst coalescer, not a cache: an entry is only reusable + * within the bucket it was built in, because the metadata carries the evaluation timestamp of the + * evaluation that built it. A later bucket builds and publishes a new entry. The default bucket is + * one millisecond, the resolution of the timestamp, so the timestamp stays exact. A wider bucket of + * {@code W} milliseconds reports the evaluations that reuse an entry up to {@code W - 1} + * milliseconds early. The {@link #UNBOUNDED} bucket leaves the timestamp out of the metadata, so + * the hooks read their own clock and the entry of a split is reusable until the slot is replaced. * *

The table is a fixed, direct-mapped array of slots. Each slot holds the metadata built last * for one key: the {@link Split}, {@link Allocation} and {@link Flag} references, the configuration - * consent to observe full evaluation data, the exposure state, and the evaluation millisecond. - * Everything else in the metadata is fixed by the split, its allocation and its flag. The key - * compares references, so a reloaded configuration, whose objects are new, never matches an entry - * of the previous one, and no metadata field can be left out of the key. + * consent to observe full evaluation data, the exposure state, and the time bucket. Everything else + * in the metadata is fixed by the split, its allocation and its flag. The key compares references, + * so a reloaded configuration, whose objects are new, never matches an entry of the previous one, + * and no metadata field can be left out of the key. * *

The table takes no lock. An entry has only final fields and is published with a single * reference store, so a reader sees either a whole entry or none. Two evaluations that miss the @@ -31,6 +35,18 @@ final class EvaluationMetadataTable { /** The largest number of slots. */ static final int MAX_SIZE = 1 << 16; + /** The default time bucket width: the millisecond resolution of the evaluation timestamp. */ + static final int DEFAULT_TIME_BUCKET_MS = 1; + + /** + * The widest time bucket: the flush interval of the flag evaluation events, which aggregate the + * first and last evaluation times of each interval. + */ + static final int MAX_TIME_BUCKET_MS = 10_000; + + /** The time bucket width that leaves the evaluation timestamp out of the metadata. */ + static final int UNBOUNDED = 0; + /** The evaluation logs no exposure, so the metadata carries no exposure state. */ static final int NO_EXPOSURE = 0; @@ -89,6 +105,18 @@ static int size(final int requestedSize) { return requestedSize == 1 ? 1 : Integer.highestOneBit(requestedSize - 1) << 1; } + /** + * @param requestedMs the requested time bucket width, in milliseconds. + * @return the width used: {@link #UNBOUNDED} for 0, at least 1 and at most {@link + * #MAX_TIME_BUCKET_MS} otherwise. + */ + static int timeBucketMs(final int requestedMs) { + if (requestedMs == UNBOUNDED) { + return UNBOUNDED; + } + return Math.max(1, Math.min(requestedMs, MAX_TIME_BUCKET_MS)); + } + /** The number of slots. */ int size() { return slots.length; @@ -106,11 +134,11 @@ ImmutableMetadata get( final Split split, final boolean observeFullEvaluationData, final int exposure, - final long evalTimestampMs) { + final long timeBucket) { final Entry entry = slots[index(split, exposure)]; if (entry != null && entry.split == split - && entry.evalTimestampMs == evalTimestampMs + && entry.timeBucket == timeBucket && entry.exposure == exposure && entry.observeFullEvaluationData == observeFullEvaluationData && entry.allocation == allocation @@ -130,22 +158,16 @@ void put( final Split split, final boolean observeFullEvaluationData, final int exposure, - final long evalTimestampMs, + final long timeBucket, final ImmutableMetadata metadata) { slots[index(split, exposure)] = new Entry( - flag, - allocation, - split, - observeFullEvaluationData, - exposure, - evalTimestampMs, - metadata); + flag, allocation, split, observeFullEvaluationData, exposure, timeBucket, metadata); } private int index(final Split split, final int exposure) { // The exposure states of a split take neighboring slots, so a subject that was already exposed - // does not evict the entry of one that was not. The millisecond is left out: the next one + // does not evict the entry of one that was not. The time bucket is left out: the next one // replaces the entry of the previous one instead of evicting another split. int hash = (System.identityHashCode(split) << 2 | exposure) * 0x9E3779B9; hash ^= hash >>> 16; @@ -159,7 +181,7 @@ private static final class Entry { final Split split; final boolean observeFullEvaluationData; final int exposure; - final long evalTimestampMs; + final long timeBucket; final ImmutableMetadata metadata; Entry( @@ -168,14 +190,14 @@ private static final class Entry { final Split split, final boolean observeFullEvaluationData, final int exposure, - final long evalTimestampMs, + final long timeBucket, final ImmutableMetadata metadata) { this.flag = flag; this.allocation = allocation; this.split = split; this.observeFullEvaluationData = observeFullEvaluationData; this.exposure = exposure; - this.evalTimestampMs = evalTimestampMs; + this.timeBucket = timeBucket; this.metadata = metadata; } } diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/FlagEvaluator.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/FlagEvaluator.java index 2a5bfb18f8a..fa5949bad42 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/FlagEvaluator.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/FlagEvaluator.java @@ -63,6 +63,7 @@ public final class FlagEvaluator { private final ExposureLookup exposureLookup; private final ExperimentationOptOutMode experimentationOptOutMode; @Nullable private final EvaluationMetadataTable metadataTable; + private final int metadataTimeBucketMs; /** * Creates an evaluator. @@ -82,7 +83,8 @@ public FlagEvaluator( spanEnrichmentEnabled, exposureLookup, experimentationOptOutMode, - EvaluationMetadataTable.DEFAULT_SIZE); + EvaluationMetadataTable.DEFAULT_SIZE, + EvaluationMetadataTable.DEFAULT_TIME_BUCKET_MS); } /** @@ -94,28 +96,36 @@ public FlagEvaluator( * @param metadataTableSize the number of slots of the table sharing the metadata of evaluations * that resolve the same split in the same millisecond, rounded up to a power of two, or 0 to * build the metadata of each evaluation. + * @param metadataTimeBucketMs the width, in milliseconds, of the time bucket in which evaluations + * of a split share their metadata and its {@code __dd_eval_timestamp_ms}, at most {@link + * EvaluationMetadataTable#MAX_TIME_BUCKET_MS}; or 0 to leave the timestamp out of the + * metadata, so the hooks read their own clock. */ public FlagEvaluator( final boolean spanEnrichmentEnabled, final ExposureLookup exposureLookup, final ExperimentationOptOutMode experimentationOptOutMode, - final int metadataTableSize) { + final int metadataTableSize, + final int metadataTimeBucketMs) { this( spanEnrichmentEnabled, exposureLookup, experimentationOptOutMode, - EvaluationMetadataTable.create(metadataTableSize)); + EvaluationMetadataTable.create(metadataTableSize), + metadataTimeBucketMs); } FlagEvaluator( final boolean spanEnrichmentEnabled, final ExposureLookup exposureLookup, final ExperimentationOptOutMode experimentationOptOutMode, - @Nullable final EvaluationMetadataTable metadataTable) { + @Nullable final EvaluationMetadataTable metadataTable, + final int metadataTimeBucketMs) { this.spanEnrichmentEnabled = spanEnrichmentEnabled; this.exposureLookup = exposureLookup; this.experimentationOptOutMode = experimentationOptOutMode; this.metadataTable = metadataTable; + this.metadataTimeBucketMs = EvaluationMetadataTable.timeBucketMs(metadataTimeBucketMs); } public ProviderEvaluation evaluate( @@ -539,26 +549,30 @@ private ProviderEvaluation resolveVariant( exposure = EvaluationMetadataTable.NEW_EXPOSURE; } // Everything else in the metadata is fixed by the split, so evaluations of the split in the - // same millisecond share it. + // same time bucket share it. + final int bucketMs = this.metadataTimeBucketMs; + final long timeBucket = + bucketMs == EvaluationMetadataTable.UNBOUNDED + ? 0 + : bucketMs == 1 ? evalTimestampMs : evalTimestampMs / bucketMs; final EvaluationMetadataTable table = this.metadataTable; ImmutableMetadata metadata = table == null ? null - : table.get( - flag, allocation, split, observeFullEvaluationData, exposure, evalTimestampMs); + : table.get(flag, allocation, split, observeFullEvaluationData, exposure, timeBucket); if (metadata == null) { metadata = buildMetadata( - flag, allocation, split, observeFullEvaluationData, doLog, exposure, evalTimestampMs); + flag, + allocation, + split, + observeFullEvaluationData, + doLog, + exposure, + bucketMs == EvaluationMetadataTable.UNBOUNDED ? null : evalTimestampMs); if (table != null) { table.put( - flag, - allocation, - split, - observeFullEvaluationData, - exposure, - evalTimestampMs, - metadata); + flag, allocation, split, observeFullEvaluationData, exposure, timeBucket, metadata); } } return ProviderEvaluation.builder() @@ -576,16 +590,20 @@ private ImmutableMetadata buildMetadata( final boolean observeFullEvaluationData, final boolean doLog, final int exposure, - final long evalTimestampMs) { - // Stamp eval-time at the resolution point so first/last_evaluation reflect evaluation time, - // not hook-fire time. Passed to the hook via provider metadata "__dd_eval_timestamp_ms". + @Nullable final Long evalTimestampMs) { final ImmutableMetadata.ImmutableMetadataBuilder metadataBuilder = ImmutableMetadata.builder() .addString("flagKey", flag.key) .addString("variationType", flag.variationType.name()) .addString(ALLOCATION_KEY, allocation.key) - .addLong(EVAL_TIMESTAMP_MS, evalTimestampMs) .addBoolean(OBSERVE_FULL_EVALUATION_DATA, observeFullEvaluationData); + // Stamp eval-time at the resolution point so first/last_evaluation reflect evaluation time, + // not hook-fire time. Passed to the hook via provider metadata "__dd_eval_timestamp_ms". + // Without + // it, the hook stamps its own time. + if (evalTimestampMs != null) { + metadataBuilder.addLong(EVAL_TIMESTAMP_MS, evalTimestampMs); + } // Surface the UFC split's serial id and the allocation's doLog flag for APM span enrichment and // for the exposure hook, which sends the exposure after the evaluation succeeds. An evaluation // that neither logs exposures nor enriches spans carries neither key. diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java index d6853ef5798..7de47026446 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java @@ -91,6 +91,7 @@ void appliesDefaults() { assertTrue(settings.isExposuresDatadogLoggingEnabled()); assertEquals("disabled", settings.experimentationOptOutMode()); assertEquals(1024, settings.evaluationMetadataTableSize()); + assertEquals(1, settings.evaluationMetadataTimeBucketMs()); } @TableTest({ @@ -196,6 +197,20 @@ void readsTheEvaluationMetadataTableSize(final String value, final int size) { .evaluationMetadataTableSize()); } + @TableTest({ + "scenario | value | width", + "unbounded | 0 | 0 ", + "custom | 100 | 100 ", + "negative | -1 | 1 ", + "invalid | ten | 1 " + }) + void readsTheEvaluationMetadataTimeBucket(final String value, final int width) { + assertEquals( + width, + TestSettings.of("feature.flags.evaluation.metadata.time.bucket.ms", value) + .evaluationMetadataTimeBucketMs()); + } + @Test void fallsBackToLowerPrioritySourcesOnBlankOrInvalidValues() { final Map properties = new HashMap<>(); diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java index 28cbf5ca8c7..e5d3deed44f 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java @@ -12,7 +12,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -195,15 +194,93 @@ void aZeroSizeBuildsTheMetadataOfEachEvaluation() { } } + @TableTest({ + "scenario | requested | width", + "unbounded | 0 | 0 ", + "negative | -5 | 1 ", + "millisecond | 1 | 1 ", + "wider | 100 | 100 ", + "capped | 60000 | 10000" + }) + void capsTheTimeBucketWidth(final int requested, final int width) { + assertEquals(width, EvaluationMetadataTable.timeBucketMs(requested)); + } + + @Test + void evaluationsInTheSameBucketShareTheMetadataOfTheFirstOne() { + final int width = 10; + final FlagEvaluator evaluator = evaluator(EvaluationMetadataTable.DEFAULT_SIZE, width); + final ServerConfiguration config = config(true, flag("flag", allocation("a", split(), true))); + // Start at the beginning of a bucket, so its later milliseconds follow. + while (System.currentTimeMillis() % width != 0) { + Thread.onSpinWait(); + } + final ImmutableMetadata first = evaluate(evaluator, config, "flag", "user"); + final long firstMs = first.getLong(EVAL_TIMESTAMP_MS); + int laterMilliseconds = 0; + long before; + ImmutableMetadata later; + while (true) { + before = System.currentTimeMillis(); + later = evaluate(evaluator, config, "flag", "user"); + if (later.getLong(EVAL_TIMESTAMP_MS) / width != firstMs / width) { + break; + } + // A later millisecond of the bucket still reports the time of the first evaluation. + assertSame(first, later); + if (before > firstMs) { + laterMilliseconds++; + } + } + + assertTrue(laterMilliseconds > 0); + assertNotSame(first, later); + assertTrue(later.getLong(EVAL_TIMESTAMP_MS) >= before); + } + @Test - void neverReturnsTheMetadataOfAnotherKeyUnderContention() throws Exception { + void anUnboundedBucketLeavesTheTimestampOutAndKeepsTheMetadata() { + final FlagEvaluator evaluator = + evaluator(EvaluationMetadataTable.DEFAULT_SIZE, EvaluationMetadataTable.UNBOUNDED); + final Flag flag = flag("flag", allocation("a", split(), true)); + final ImmutableMetadata first = evaluate(evaluator, config(true, flag), "flag", "user"); + final long start = System.currentTimeMillis(); + + assertNull(first.getLong(EVAL_TIMESTAMP_MS)); + assertEquals("a", first.getString(ALLOCATION_KEY)); + while (System.currentTimeMillis() < start + 3) { + assertSame(first, evaluate(evaluator, config(true, flag), "flag", "user")); + } + assertNotSame(first, evaluate(evaluator, config(false, flag), "flag", "user")); + } + + @Test + void anUnboundedBucketWithoutTableBuildsTimestamplessMetadata() { + final FlagEvaluator evaluator = evaluator(0, EvaluationMetadataTable.UNBOUNDED); + final ServerConfiguration config = config(true, flag("flag", allocation("a", split(), true))); + final ImmutableMetadata first = evaluate(evaluator, config, "flag", "user"); + final ImmutableMetadata second = evaluate(evaluator, config, "flag", "user"); + + assertNotSame(first, second); + assertNull(first.getLong(EVAL_TIMESTAMP_MS)); + assertEquals(first.asUnmodifiableMap(), second.asUnmodifiableMap()); + } + + @TableTest({ + "scenario | timeBucketMs", + "millisecond | 1 ", + "wider bucket | 10 ", + "unbounded | 0 " + }) + void neverReturnsTheMetadataOfAnotherKeyUnderContention(final int timeBucketMs) throws Exception { // A single slot makes every key collide. final FlagEvaluator evaluator = new FlagEvaluator( false, exposedUser(), ExperimentationOptOutMode.DISABLED, - EvaluationMetadataTable.create(1)); + EvaluationMetadataTable.create(1), + timeBucketMs); final List flags = new ArrayList<>(); for (int i = 0; i < 4; i++) { flags.add(flag("flag-" + i, allocation("allocation-" + i, split(), i % 2 == 0))); @@ -234,8 +311,16 @@ void neverReturnsTheMetadataOfAnotherKeyUnderContention() throws Exception { assertEquals( flag % 2 == 0 ? "exposed".equals(subject) : null, metadata.getBoolean(EXPOSURE_CACHE_HIT)); - final long ms = metadata.getLong(EVAL_TIMESTAMP_MS); - assertTrue(before <= ms && ms <= after, ms + " not in " + before + ".." + after); + if (timeBucketMs == EvaluationMetadataTable.UNBOUNDED) { + assertNull(metadata.getLong(EVAL_TIMESTAMP_MS)); + } else { + // A reused entry reports the time of the evaluation that built it, in the same + // bucket. + final long ms = metadata.getLong(EVAL_TIMESTAMP_MS); + final long earliest = before - before % timeBucketMs; + assertTrue( + earliest <= ms && ms <= after, ms + " not in " + earliest + ".." + after); + } } return null; })); @@ -250,7 +335,12 @@ void neverReturnsTheMetadataOfAnotherKeyUnderContention() throws Exception { } private static FlagEvaluator evaluator(final int tableSize) { - return new FlagEvaluator(false, exposedUser(), ExperimentationOptOutMode.DISABLED, tableSize); + return evaluator(tableSize, EvaluationMetadataTable.DEFAULT_TIME_BUCKET_MS); + } + + private static FlagEvaluator evaluator(final int tableSize, final int timeBucketMs) { + return new FlagEvaluator( + false, exposedUser(), ExperimentationOptOutMode.DISABLED, tableSize, timeBucketMs); } private static FlagEvaluator.ExposureLookup exposedUser() { @@ -266,7 +356,6 @@ private static ImmutableMetadata evaluate( evaluator .evaluate(config, Integer.class, flag, 0, new MutableContext(subject)) .getFlagMetadata(); - assertNotNull(metadata.getLong(EVAL_TIMESTAMP_MS)); return metadata; } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index e52012c2ed3..5df55592c12 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -45,5 +45,15 @@ public final class FeatureFlaggingConfig { public static final String FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE = "feature.flags.evaluation.metadata.table.size"; + /** + * The width, in milliseconds, of the time bucket in which evaluations of a split share their flag + * metadata, including its evaluation timestamp, so evaluations report up to the width minus one + * millisecond early. At most {@code 10000}. {@code 0} leaves the timestamp out of the metadata, + * so the hooks read their own clock. Default: {@code 1}, which keeps the timestamps exact. Maps + * to {@code DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS}. + */ + public static final String FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS = + "feature.flags.evaluation.metadata.time.bucket.ms"; + private FeatureFlaggingConfig() {} } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index f23c71d6881..82cd50a31ce 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1620,6 +1620,14 @@ "aliases": [] } ], + "DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS": [ + { + "version": "A", + "type": "int", + "default": "1", + "aliases": [] + } + ], "DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED": [ { "version": "A", From 99df09ebd27d2194385764aafd66c4597333b745 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 21 Nov 2026 14:46:34 +0000 Subject: [PATCH 18/18] Default the shared flag metadata time bucket to 10 ms The sweep in EvaluationMetadataTableBenchmark shows a 10 ms bucket shares most of what wider buckets share: at 4,096 slots on one thread, the hit rate rises from 52% to 85% over 1 ms, against at most 4 more points for 100 ms or more. Evaluations then report first_evaluation and last_evaluation at most 9 ms early in a 10 s flush interval. A 1 ms bucket still keeps the timestamps exact. Co-authored-by: Douglas Q Hawkins --- dd-openfeature/README.md | 2 +- .../openfeature/internal/config/Settings.java | 6 +++--- .../evaluation/EvaluationMetadataTable.java | 16 +++++++++------- .../internal/config/SettingsTest.java | 7 ++++--- .../evaluation/EvaluationMetadataTableTest.java | 3 ++- .../trace/api/config/FeatureFlaggingConfig.java | 4 ++-- metadata/supported-configurations.json | 2 +- 7 files changed, 22 insertions(+), 18 deletions(-) diff --git a/dd-openfeature/README.md b/dd-openfeature/README.md index 2612478c10b..cf23df8f97a 100644 --- a/dd-openfeature/README.md +++ b/dd-openfeature/README.md @@ -55,7 +55,7 @@ Settings are read from the Datadog Java agent configuration when attached, then | `DD_FEATURE_FLAGS_EXPERIMENTATION_OPT_OUT_MODE` | `disabled` | Controls per-subject experimentation opt-out: `disabled`, `immediate`, or `future_experiments_only`. See [Per-subject experimentation opt-out](#per-subject-experimentation-opt-out). | | `DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED` | `false` | Lets the `dd_evaluation_telemetry_opt_out` context attribute opt subjects out of evaluation telemetry. See [Per-subject evaluation telemetry opt-out](#per-subject-evaluation-telemetry-opt-out). | | `DD_FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE` | `1024` | Slots of the table that lets evaluations resolving the same split in the same millisecond share their flag metadata, rounded up to a power of two (at most 65536). `0` builds the metadata of every evaluation. | -| `DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS` | `1` | Width, in milliseconds, of the time bucket in which evaluations of the same split share their flag metadata (at most 10000). The `first_evaluation` and `last_evaluation` times of flag evaluation events can then be up to the width minus one millisecond early; `1` keeps them exact. `0` leaves the time out of the shared metadata, and the evaluation events use the time their hook runs. | +| `DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS` | `10` | Width, in milliseconds, of the time bucket in which evaluations of the same split share their flag metadata (at most 10000). The `first_evaluation` and `last_evaluation` times of flag evaluation events can then be up to the width minus one millisecond early; `1` keeps them exact. `0` leaves the time out of the shared metadata, and the evaluation events use the time their hook runs. | | `DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED` | `false` | Adds flag evaluations to the local root span. Requires `dd-java-agent`. | | `DD_API_KEY`, `DD_SITE`, `DD_ENV`, `DD_SERVICE`, `DD_VERSION` | | Credentials and event context. | | `DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED` | | Deprecated: `true` selects `remote_config`, `false` disables Feature Flags. An explicit source wins. | diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java index 854821d299c..2e4d4449d75 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/config/Settings.java @@ -54,7 +54,7 @@ public final class Settings { static final int DEFAULT_POLL_INTERVAL_SECONDS = 30; static final int DEFAULT_REQUEST_TIMEOUT_SECONDS = 5; static final int DEFAULT_EVALUATION_METADATA_TABLE_SIZE = 1024; - static final int DEFAULT_EVALUATION_METADATA_TIME_BUCKET_MS = 1; + static final int DEFAULT_EVALUATION_METADATA_TIME_BUCKET_MS = 10; private final AgentBridge bridge; private final UnaryOperator systemProperties; @@ -205,8 +205,8 @@ public int evaluationMetadataTableSize() { * evaluation that built it, up to the width minus one millisecond early. The evaluator caps it at * 10,000 milliseconds, the flush interval of the flag evaluation events. * - * @return the width in milliseconds (1 by default, which keeps the timestamps exact), or 0 to - * leave the timestamp out of the metadata, so the hooks read their own clock. + * @return the width in milliseconds (10 by default; 1 keeps the timestamps exact), or 0 to leave + * the timestamp out of the metadata, so the hooks read their own clock. */ public int evaluationMetadataTimeBucketMs() { return getNonNegativeInt( diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java index a9da7fc5d7e..e88475123cc 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTable.java @@ -11,11 +11,13 @@ * Shares the flag metadata of successful evaluations that resolve the same split in the same time * bucket. With a bounded bucket, this is a burst coalescer, not a cache: an entry is only reusable * within the bucket it was built in, because the metadata carries the evaluation timestamp of the - * evaluation that built it. A later bucket builds and publishes a new entry. The default bucket is - * one millisecond, the resolution of the timestamp, so the timestamp stays exact. A wider bucket of - * {@code W} milliseconds reports the evaluations that reuse an entry up to {@code W - 1} - * milliseconds early. The {@link #UNBOUNDED} bucket leaves the timestamp out of the metadata, so - * the hooks read their own clock and the entry of a split is reusable until the slot is replaced. + * evaluation that built it. A later bucket builds and publishes a new entry. A bucket of {@code W} + * milliseconds reports the evaluations that reuse an entry up to {@code W - 1} milliseconds early: + * the default of 10 milliseconds, at most 9 milliseconds early in the 10 second flush interval of + * the flag evaluation events, shares most of what wider buckets share. A bucket of one millisecond, + * the resolution of the timestamp, keeps the timestamp exact. The {@link #UNBOUNDED} bucket leaves + * the timestamp out of the metadata, so the hooks read their own clock and the entry of a split is + * reusable until the slot is replaced. * *

The table is a fixed, direct-mapped array of slots. Each slot holds the metadata built last * for one key: the {@link Split}, {@link Allocation} and {@link Flag} references, the configuration @@ -35,8 +37,8 @@ final class EvaluationMetadataTable { /** The largest number of slots. */ static final int MAX_SIZE = 1 << 16; - /** The default time bucket width: the millisecond resolution of the evaluation timestamp. */ - static final int DEFAULT_TIME_BUCKET_MS = 1; + /** The default time bucket width. */ + static final int DEFAULT_TIME_BUCKET_MS = 10; /** * The widest time bucket: the flush interval of the flag evaluation events, which aggregate the diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java index 7de47026446..6e4bb603725 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/config/SettingsTest.java @@ -91,7 +91,7 @@ void appliesDefaults() { assertTrue(settings.isExposuresDatadogLoggingEnabled()); assertEquals("disabled", settings.experimentationOptOutMode()); assertEquals(1024, settings.evaluationMetadataTableSize()); - assertEquals(1, settings.evaluationMetadataTimeBucketMs()); + assertEquals(10, settings.evaluationMetadataTimeBucketMs()); } @TableTest({ @@ -201,8 +201,9 @@ void readsTheEvaluationMetadataTableSize(final String value, final int size) { "scenario | value | width", "unbounded | 0 | 0 ", "custom | 100 | 100 ", - "negative | -1 | 1 ", - "invalid | ten | 1 " + "exact | 1 | 1 ", + "negative | -1 | 10 ", + "invalid | ten | 10 " }) void readsTheEvaluationMetadataTimeBucket(final String value, final int width) { assertEquals( diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java index 0f605de2a55..1104690efec 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/internal/evaluation/EvaluationMetadataTableTest.java @@ -353,8 +353,9 @@ void neverReturnsTheMetadataOfAnotherKeyUnderContention(final int timeBucketMs) } } + /** An evaluator with a one millisecond bucket, which keeps the timestamps exact. */ private static FlagEvaluator evaluator(final int tableSize) { - return evaluator(tableSize, EvaluationMetadataTable.DEFAULT_TIME_BUCKET_MS); + return evaluator(tableSize, 1); } private static FlagEvaluator evaluator(final int tableSize, final int timeBucketMs) { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 5df55592c12..1e38da77652 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -49,8 +49,8 @@ public final class FeatureFlaggingConfig { * The width, in milliseconds, of the time bucket in which evaluations of a split share their flag * metadata, including its evaluation timestamp, so evaluations report up to the width minus one * millisecond early. At most {@code 10000}. {@code 0} leaves the timestamp out of the metadata, - * so the hooks read their own clock. Default: {@code 1}, which keeps the timestamps exact. Maps - * to {@code DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS}. + * so the hooks read their own clock, and {@code 1} keeps the timestamps exact. Default: {@code + * 10}. Maps to {@code DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS}. */ public static final String FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS = "feature.flags.evaluation.metadata.time.bucket.ms"; diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 82cd50a31ce..7f349ae6942 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1624,7 +1624,7 @@ { "version": "A", "type": "int", - "default": "1", + "default": "10", "aliases": [] } ],