Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
92ed06e
Benchmark the FFE scale evaluation suite
leoromanovsky Nov 17, 2026
c18784a
Benchmark every scale dimension on master, and the configuration load
leoromanovsky Nov 17, 2026
a887657
Check the scale suites in the unit tests and harden the benchmarks
leoromanovsky Nov 17, 2026
09289ff
Pin the fixtures to the latest head of the scale suite PR
leoromanovsky Nov 17, 2026
30df999
Benchmark one-of lists of small numbers
leoromanovsky Nov 17, 2026
8d0f39c
Benchmark the scale cases with the client's layered context
leoromanovsky Nov 17, 2026
72d60ca
Pin the fixtures with a 4 MB largest JSON value
leoromanovsky Nov 17, 2026
e69a0af
Give each benchmark thread its own case cursor
leoromanovsky Nov 17, 2026
911924f
Pin the fixtures with first-match cases and check selection metadata
leoromanovsky Nov 17, 2026
89a6035
Pin the fixtures to the merged scale suites
leoromanovsky Nov 18, 2026
89120c7
Move the shared fixtures into dd-openfeature
leoromanovsky Nov 18, 2026
6f3c1dd
Ignore the JVM settings in the scale suites
leoromanovsky Nov 18, 2026
9b7f7de
Parse each scale suite once in the test
leoromanovsky Nov 18, 2026
109bd9f
Check the benchmark's own case-file list
leoromanovsky Nov 18, 2026
53243ac
Merge branch 'master' into leo.romanovsky/ffe-scale-suite-benchmark
leoromanovsky Nov 19, 2026
971fcdf
Parenthesize the cursor wrap condition
leoromanovsky Nov 19, 2026
d27533f
Share flag metadata across evaluations of a split in the same millise…
leoromanovsky Nov 19, 2026
bd66ac9
Merge branch 'master' into leo.romanovsky/ffe-metadata-table-78f95e
leoromanovsky Nov 19, 2026
136f3eb
Make the time bucket of the shared flag metadata configurable
leoromanovsky Nov 19, 2026
159c6a5
Merge branch 'master' into leo.romanovsky/ffe-metadata-table
leoromanovsky Nov 21, 2026
99df09e
Default the shared flag metadata time bucket to 10 ms
leoromanovsky Nov 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions dd-openfeature/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ Settings are read from the Datadog Java agent configuration when attached, then
| `DD_FEATURE_FLAGS_EXPOSURES_DATADOG_LOGGING_ENABLED` | `true` | Sends exposures to Datadog. See [Stop sending exposures to Datadog](#stop-sending-exposures-to-datadog). |
| `DD_FEATURE_FLAGS_EXPERIMENTATION_OPT_OUT_MODE` | `disabled` | Controls per-subject experimentation opt-out: `disabled`, `immediate`, or `future_experiments_only`. See [Per-subject experimentation opt-out](#per-subject-experimentation-opt-out). |
| `DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED` | `false` | Lets the `dd_evaluation_telemetry_opt_out` context attribute opt subjects out of evaluation telemetry. See [Per-subject evaluation telemetry opt-out](#per-subject-evaluation-telemetry-opt-out). |
| `DD_FEATURE_FLAGS_EVALUATION_METADATA_TABLE_SIZE` | `1024` | Slots of the table that lets evaluations resolving the same split in the same millisecond share their flag metadata, rounded up to a power of two (at most 65536). `0` builds the metadata of every evaluation. |
| `DD_FEATURE_FLAGS_EVALUATION_METADATA_TIME_BUCKET_MS` | `10` | Width, in milliseconds, of the time bucket in which evaluations of the same split share their flag metadata (at most 10000). The `first_evaluation` and `last_evaluation` times of flag evaluation events can then be up to the width minus one millisecond early; `1` keeps them exact. `0` leaves the time out of the shared metadata, and the evaluation events use the time their hook runs. |
| `DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED` | `false` | Adds flag evaluations to the local root span. Requires `dd-java-agent`. |
| `DD_API_KEY`, `DD_SITE`, `DD_ENV`, `DD_SERVICE`, `DD_VERSION` | | Credentials and event context. |
| `DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED` | | Deprecated: `true` selects `remote_config`, `false` disables Feature Flags. An explicit source wins. |
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,271 @@
package com.datadog.openfeature.internal.evaluation;

import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVAL_TIMESTAMP_MS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;

import com.datadog.openfeature.ExperimentationOptOutMode;
import com.datadog.openfeature.internal.ufc.ServerConfiguration;
import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser;
import de.thetaphi.forbiddenapis.SuppressForbidden;
import dev.openfeature.sdk.EvaluationContext;
import dev.openfeature.sdk.ImmutableContext;
import dev.openfeature.sdk.ProviderEvaluation;
import dev.openfeature.sdk.Value;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.SplittableRandom;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.LongAdder;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;

/**
* Requests that each evaluate {@code flagsPerRequest} flags drawn from 1,000 enabled flags of the
* 10,000-flag scale configuration ({@code evaluation-suites/scale}), for a subject and cohort that
* change on every request, on threads sharing one evaluator. It sweeps the evaluation metadata
* table size and time bucket width. {@code tableSize=0} builds the metadata of every evaluation, as
* without the table; {@code timeBucketMs=0} leaves the timestamp out of the metadata.
*
* <p>{@code popularity=uniform} draws each flag of a request uniformly; {@code skewed} draws it
* from a Zipf distribution with exponent 1, so a few flags take most evaluations. An operation is
* one request, sampled for its mean and percentiles. Each measurement iteration prints the table's
* hit rate, the share of successful evaluations that reused metadata, and how early the reported
* {@code __dd_eval_timestamp_ms} was: the clock just before the evaluation minus the reported time,
* sampled on every 8th request. Without a timestamp in the metadata, the flag evaluation hook reads
* its own clock after the evaluation, so the reported time is never early.
*
* <p>Run with the thread count as an option: {@code java -jar dd-openfeature/build/libs/*-jmh.jar
* EvaluationMetadataTableBenchmark -t 12 -prof gc}. {@code tableSize=0} ignores the bucket width.
*/
@State(Scope.Benchmark)
@Warmup(iterations = 2, time = 1, timeUnit = SECONDS)
@Measurement(iterations = 3, time = 1, timeUnit = SECONDS)
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(NANOSECONDS)
@Fork(value = 1)
public class EvaluationMetadataTableBenchmark {
private static final String CONFIG_PATH =
"dd-openfeature/src/test/fixtures/ffe-system-test-data/evaluation-suites/scale/ufc-config.json";
private static final int POPULATION = 1000;
private static final int CONTEXTS = 1024;
private static final int DRAWS = 1 << 16;
private static final int ERROR_SAMPLE_MASK = 7;
private static final int MAX_ERROR_MS = 10_000;

/** The number of flags each request evaluates. */
@Param({"10", "50"})
public int flagsPerRequest;

/** The requested table size, 0 for no table. */
@Param({"0", "256", "1024", "4096"})
public int tableSize;

/** The time bucket width in milliseconds, 0 to leave the timestamp out of the metadata. */
@Param({"1", "10", "100", "1000", "0"})
public int timeBucketMs;

/** How the flags of a request are drawn: {@code uniform} or {@code skewed} (Zipf). */
@Param({"uniform", "skewed"})
public String popularity;

private ServerConfiguration config;
private FlagEvaluator evaluator;
private String[] flags;
private double[] cumulative;
private EvaluationContext[] contexts;
private final AtomicInteger threadSeeds = new AtomicInteger();
private final List<Requests> threads = Collections.synchronizedList(new ArrayList<>());
private final LongAdder misses = new LongAdder();
private final LongAdder lookups = new LongAdder();

/** One benchmark thread: its flag draws, its positions, and its timestamp error histogram. */
@State(Scope.Thread)
public static class Requests {
private int[] draws;
private int nextDraw;
private int nextContext;
private int requests;
private long[] earlyMs;

@Setup(Level.Trial)
public void setUp(final EvaluationMetadataTableBenchmark benchmark) {
final int seed = benchmark.threadSeeds.incrementAndGet();
draws = benchmark.draws(new SplittableRandom(seed));
nextContext = (seed * 97) & (CONTEXTS - 1);
earlyMs = new long[MAX_ERROR_MS + 1];
benchmark.threads.add(this);
}

@Setup(Level.Iteration)
public void reset() {
Arrays.fill(earlyMs, 0);
}

int nextDraw() {
final int draw = draws[nextDraw];
nextDraw = (nextDraw + 1) & (DRAWS - 1);
return draw;
}

EvaluationContext nextContext(final EvaluationContext[] contexts) {
final EvaluationContext context = contexts[nextContext];
nextContext = (nextContext + 1) & (CONTEXTS - 1);
return context;
}
}

@Setup(Level.Trial)
public void setUp() throws IOException {
config = UniversalFlagConfigParser.parse(Files.readAllBytes(configFile()));
evaluator =
new FlagEvaluator(
false,
(flag, subject, allocation, variant, serialId) -> false,
ExperimentationOptOutMode.DISABLED,
true,
EvaluationMetadataTable.create(tableSize, misses),
timeBucketMs);
// Every 10th bulk flag is disabled: skip them, so each evaluation resolves a split.
flags = new String[POPULATION];
for (int i = 0, flag = 0; i < POPULATION; flag++) {
if (flag % 10 != 9) {
flags[i++] = String.format(Locale.ROOT, "scale-bulk-%05d", flag);
}
}
cumulative = new double[POPULATION];
double sum = 0;
for (int i = 0; i < POPULATION; i++) {
sum += "skewed".equals(popularity) ? 1.0 / (i + 1) : 1.0;
cumulative[i] = sum;
}
for (int i = 0; i < POPULATION; i++) {
cumulative[i] /= sum;
}
contexts = new EvaluationContext[CONTEXTS];
for (int i = 0; i < CONTEXTS; i++) {
contexts[i] =
new ImmutableContext(
"subject-" + i,
Collections.singletonMap(
"cohort", new Value(String.format(Locale.ROOT, "cohort-%02d", i % 100))));
}
for (final String flag : flags) {
final ProviderEvaluation<Boolean> result =
evaluator.evaluate(config, Boolean.class, flag, false, contexts[0]);
if (result.getErrorCode() != null || result.getVariant() == null) {
throw new IllegalStateException(flag + " did not resolve a split: " + result.getReason());
}
}
}

int[] draws(final SplittableRandom random) {
final int[] draws = new int[DRAWS];
for (int i = 0; i < DRAWS; i++) {
final int index = Arrays.binarySearch(cumulative, random.nextDouble());
draws[i] = Math.min(index < 0 ? -index - 1 : index, POPULATION - 1);
}
return draws;
}

@Setup(Level.Iteration)
public void startIteration() {
misses.reset();
lookups.reset();
}

@TearDown(Level.Iteration)
@SuppressForbidden
public void reportIteration() {
final long total = lookups.sum();
final long missed = tableSize == 0 ? total : misses.sum();
final long[] early = new long[MAX_ERROR_MS + 1];
synchronized (threads) {
for (final Requests thread : threads) {
for (int i = 0; i <= MAX_ERROR_MS; i++) {
early[i] += thread.earlyMs[i];
}
}
}
long samples = 0;
int max = 0;
for (int i = 0; i <= MAX_ERROR_MS; i++) {
samples += early[i];
if (early[i] != 0) {
max = i;
}
}
int p99 = 0;
for (long seen = early[0]; seen < samples * 0.99 && p99 < MAX_ERROR_MS; ) {
seen += early[++p99];
}
System.out.printf(
Locale.ROOT,
"%n@@ table=%d bucket=%d flags=%d popularity=%s lookups=%d misses=%d hitRate=%.4f"
+ " earlyP99Ms=%d earlyMaxMs=%d samples=%d%n",
EvaluationMetadataTable.size(tableSize),
timeBucketMs,
flagsPerRequest,
popularity,
total,
missed,
total == 0 ? 0 : (double) (total - missed) / total,
p99,
max,
samples);
}

@Benchmark
public void request(final Requests requests, final Blackhole blackhole) {
final EvaluationContext context = requests.nextContext(contexts);
final boolean sample = (requests.requests++ & ERROR_SAMPLE_MASK) == 0;
for (int i = 0; i < flagsPerRequest; i++) {
final String flag = flags[requests.nextDraw()];
if (sample) {
final long before = System.currentTimeMillis();
final ProviderEvaluation<Boolean> result =
evaluator.evaluate(config, Boolean.class, flag, false, context);
final Long reported = result.getFlagMetadata().getLong(EVAL_TIMESTAMP_MS);
// Without the timestamp, the hook reads its clock after the evaluation: never early.
final long early = reported == null ? 0 : before - reported;
requests.earlyMs[(int) Math.max(0, Math.min(early, MAX_ERROR_MS))]++;
blackhole.consume(result);
} else {
blackhole.consume(evaluator.evaluate(config, Boolean.class, flag, false, context));
}
}
lookups.add(flagsPerRequest);
}

private static Path configFile() {
Path directory = Paths.get("").toAbsolutePath();
while (directory != null) {
final Path candidate = directory.resolve(CONFIG_PATH);
if (Files.exists(candidate)) {
return candidate;
}
directory = directory.getParent();
}
throw new IllegalStateException(
"Unable to find the FFE scale suite: initialize the ffe-system-test-data submodule");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,12 @@ class DDEvaluator implements Evaluator, Consumer<ServerConfiguration> {
// hook.
this.core =
new FlagEvaluator(
spanEnrichmentEnabled, this::isExposed, experimentationOptOutMode, exposuresEnabled);
spanEnrichmentEnabled,
this::isExposed,
experimentationOptOutMode,
exposuresEnabled,
settings.evaluationMetadataTableSize(),
settings.evaluationMetadataTimeBucketMs());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ public final class Settings {
static final String EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED =
"feature.flags.evaluation.telemetry.per.subject.opt.out.enabled";
static final String EXPERIMENTATION_OPT_OUT_MODE = "feature.flags.experimentation.opt.out.mode";
static final String EVALUATION_METADATA_TABLE_SIZE =
"feature.flags.evaluation.metadata.table.size";
static final String EVALUATION_METADATA_TIME_BUCKET_MS =
"feature.flags.evaluation.metadata.time.bucket.ms";
static final String TRACE_DEBUG = "trace.debug";
static final String API_KEY = "api-key";
static final String SITE = "site";
Expand All @@ -49,6 +53,8 @@ public final class Settings {
static final String DEFAULT_SITE = "datadoghq.com";
static final int DEFAULT_POLL_INTERVAL_SECONDS = 30;
static final int DEFAULT_REQUEST_TIMEOUT_SECONDS = 5;
static final int DEFAULT_EVALUATION_METADATA_TABLE_SIZE = 1024;
static final int DEFAULT_EVALUATION_METADATA_TIME_BUCKET_MS = 10;

private final AgentBridge bridge;
private final UnaryOperator<String> systemProperties;
Expand Down Expand Up @@ -182,6 +188,31 @@ public String experimentationOptOutMode() {
return mode == null ? "disabled" : mode;
}

/**
* The number of slots of the table sharing the flag metadata of evaluations that resolve the same
* split in the same millisecond. The evaluator rounds it up to a power of two.
*
* @return the number of slots, or 0 to build the metadata of each evaluation (1024 by default).
*/
public int evaluationMetadataTableSize() {
return getNonNegativeInt(
EVALUATION_METADATA_TABLE_SIZE, DEFAULT_EVALUATION_METADATA_TABLE_SIZE);
}

/**
* The width of the time bucket in which evaluations of a split share their flag metadata,
* including its evaluation timestamp. Evaluations that reuse the metadata report the time of the
* evaluation that built it, up to the width minus one millisecond early. The evaluator caps it at
* 10,000 milliseconds, the flush interval of the flag evaluation events.
*
* @return the width in milliseconds (10 by default; 1 keeps the timestamps exact), or 0 to leave
* the timestamp out of the metadata, so the hooks read their own clock.
*/
public int evaluationMetadataTimeBucketMs() {
return getNonNegativeInt(
EVALUATION_METADATA_TIME_BUCKET_MS, DEFAULT_EVALUATION_METADATA_TIME_BUCKET_MS);
}

/**
* Tells whether {@code DD_TRACE_DEBUG} is on. It gates the diagnostics that log raw evaluation
* contexts and values, on top of the application's logging configuration.
Expand Down Expand Up @@ -292,17 +323,27 @@ private Boolean getBoolean(final String key) {

/** Reads the first value that parses, in source priority order, like the agent. */
private int getPositiveInt(final String key, final int defaultValue) {
return getInt(key, defaultValue, 1, "positive");
}

/** Reads the first value that parses, in source priority order, like the agent. */
private int getNonNegativeInt(final String key, final int defaultValue) {
return getInt(key, defaultValue, 0, "zero or positive");
}

private int getInt(
final String key, final int defaultValue, final int minimum, final String expected) {
for (int source = 0; source < SOURCE_COUNT; source++) {
final String value = value(key, source);
if (nonBlank(value) == null) {
continue;
}
try {
final int parsed = Integer.parseInt(value);
if (parsed > 0) {
if (parsed >= minimum) {
return parsed;
}
LOGGER.warn("Invalid value for {}: {}. The value must be positive", key, value);
LOGGER.warn("Invalid value for {}: {}. The value must be {}", key, value, expected);
return defaultValue;
} catch (final NumberFormatException e) {
LOGGER.warn("Invalid value for {}: {}", key, value);
Expand Down
Loading