Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ updates:
schedule:
interval: weekly
allow:
- dependency-name: dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data
- dependency-name: dd-openfeature/src/test/fixtures/ffe-system-test-data
labels:
- 'comp: openfeature'
- 'tag: dependencies'
Expand Down
4 changes: 2 additions & 2 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[submodule "dd-java-agent/agent-jmxfetch/integrations-core"]
path = dd-java-agent/agent-jmxfetch/integrations-core
url = https://github.com/DataDog/integrations-core.git
[submodule "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data"]
path = dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data
[submodule "dd-openfeature/src/test/fixtures/ffe-system-test-data"]
path = dd-openfeature/src/test/fixtures/ffe-system-test-data
url = https://github.com/DataDog/ffe-system-test-data.git
2 changes: 1 addition & 1 deletion BUILDING.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ winget install --id Docker.DockerDesktop

> [!TIP]
> This will keep the submodules in `dd-java-agent/agent-jmxfetch/integrations-core` and
> `dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data` up-to-date.
> `dd-openfeature/src/test/fixtures/ffe-system-test-data` up-to-date.
> There is also an automated check when opening a pull request if you are trying to submit a module version change (usually an outdated version).

> [!NOTE]
Expand Down
30 changes: 19 additions & 11 deletions dd-openfeature/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -142,16 +142,24 @@ tasks.named("check") {
dependsOn(japicmpPublicApi)
}

// These suites read shared fixtures directly from the smoke-test resources.
// The shared FFE fixtures live outside any source set, so Gradle neither copies nor packages them:
// the tests and benchmarks read them from disk, and the smoke tests copy the files they need.
val fixtures = layout.projectDirectory.dir("src/test/fixtures")

tasks.withType<Test>().configureEach {
inputs.dir(
rootProject.layout.projectDirectory.dir(
"dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data"
)
).withPathSensitivity(PathSensitivity.RELATIVE)
inputs.dir(
rootProject.layout.projectDirectory.dir(
"dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out"
)
).withPathSensitivity(PathSensitivity.RELATIVE)
inputs.dir(fixtures).withPropertyName("fixtures").withPathSensitivity(PathSensitivity.RELATIVE)
}

// ScaleSuiteTest checks the case files that ScaleSuiteBenchmark lists in its @Param. It reads the
// annotation by reflection, as the test source set cannot compile against the jmh one.
tasks.named<Test>("test") {
classpath += sourceSets["jmh"].output + configurations["jmhRuntimeClasspath"]
}

// The scale benchmarks read the fixtures from disk at run time, so a fixture change reruns them. A
// file tree rather than a directory: the suites are missing when the submodule is not initialized.
tasks.named("jmh") {
inputs.files(
fixtures.dir("ffe-system-test-data").asFileTree.matching { include("evaluation-suites/**") }
).withPropertyName("scaleSuites").withPathSensitivity(PathSensitivity.RELATIVE)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package com.datadog.openfeature;

import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;

import de.thetaphi.forbiddenapis.SuppressForbidden;
import dev.openfeature.sdk.ImmutableContext;
import dev.openfeature.sdk.LayeredEvaluationContext;
import dev.openfeature.sdk.ProviderEvaluation;
import java.io.IOException;
import java.util.List;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;

/**
* Evaluation cost on the opt-in scale suites of the shared FFE fixtures ({@code
* evaluation-suites/scale} and {@code evaluation-suites/scale-json}): large lists, regex, rules,
* splits and shards, time windows, many variations and flags, and JSON values up to 4 MB. The
* dimensions are listed in {@code evaluation-suites/scale/AGENTS.md} of the fixtures.
*
* <p>Each operation evaluates a case of the selected case file through the provider. With {@code
* index=all}, each benchmark thread goes through the cases of the file in turn, so a row is the
* unweighted mean over every case of the file, and its slowest case dominates it. Select one case
* to measure one shape: the setup prints the index of each case.
*
* <p>The setup evaluates every case once and the teardown again after the measurement. Both fail if
* a value, reason, or expected selection metadata (serial ID, {@code doLog}, split features)
* differs from the expected one, or if an evaluation reports an error, so a run that resolves a
* case wrongly fails instead of reporting a number. JSON values are compared by numeric value: a
* JSON integer can come back as an integer or as a double. This check guards the measurement and is
* not the conformance check: {@code CanonicalFixtureTest} checks the types.
*
* <p>The provider is called directly: the OpenFeature client, its hooks and exposure logging are
* not measured. {@code evaluate} passes a reused context. {@code evaluateLayered} passes a fresh
* {@code LayeredEvaluationContext} around it on each call, as the client does, so it also shows the
* cost of reading attributes through the client's context, such as a rule on {@code id} when the
* context has no {@code id}. Select one method with {@code
* -Pjmh.includes=ScaleSuiteBenchmark.evaluate$}.
*
* <p>The cases and their check are shared with {@code ScaleSuiteTest}, which runs them in the unit
* tests. One fork is enough for allocation and for large time differences. For a time difference
* under about 15%, run three forks: {@code -Pjmh.forks=3} or {@code -f 3}.
*
* <p>Run all case files: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteBenchmark
* -Pjmh.profilers=gc}. Run one case: {@code ./gradlew :dd-openfeature:jmhJar}, then {@code java
* -jar dd-openfeature/build/libs/*-jmh.jar ScaleSuiteBenchmark -p cases=scale/splits -p index=0
* -prof gc}.
*/
@State(Scope.Benchmark)
Comment thread
leoromanovsky marked this conversation as resolved.
Resolved
@Warmup(iterations = 3, time = 2, timeUnit = SECONDS)
@Measurement(iterations = 5, time = 2, timeUnit = SECONDS)
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(NANOSECONDS)
@Fork(value = 1)
public class ScaleSuiteBenchmark {

/**
* A case file, as {@code <suite>/<file>} without the {@code .json} extension. {@code
* ScaleSuiteTest} reads this list and fails unless it names every case file of the fixtures, in
* sorted order.
*/
@Param({
"scale/flags",
"scale/numeric",
"scale/one-of",
"scale/one-of-many",
"scale/one-of-small-numbers",
"scale/regex",
"scale/regex-many",
"scale/rules",
"scale/rules-sweep",
"scale/semver",
"scale/splits",
"scale/time-windows",
"scale/variations",
"scale/wide-context",
"scale-json/json-shapes",
"scale-json/json-sizes",
"scale-json/json-values"
})
public String cases;

/** The index of one case in the file, or {@code all} to go through every case in turn. */
@Param({"all"})
public String index;

private Provider provider;
private ScaleCase[] scaleCases;

/** The position of one benchmark thread in the cases, so threads share no mutable state. */
@State(Scope.Thread)
public static class Cursor {
private int next;

ScaleCase next(final ScaleCase[] scaleCases) {
final ScaleCase scaleCase = scaleCases[next];
next = (next + 1 == scaleCases.length) ? 0 : next + 1;
return scaleCase;
}
}

@Setup(Level.Trial)
@SuppressForbidden
public void setUp() throws IOException {
final int slash = cases.indexOf('/');
final ScaleSuites.Suite suite = ScaleSuites.load(cases.substring(0, slash));
provider = suite.provider();
final List<ScaleCase> loaded = suite.cases(cases.substring(slash + 1));
for (int i = 0; i < loaded.size(); i++) {
final ScaleCase scaleCase = loaded.get(i);
scaleCase.check(scaleCase.evaluate(provider));
System.out.println("[" + cases + "] case " + i + ": " + scaleCase.describe());
}
scaleCases =
"all".equals(index)
? loaded.toArray(new ScaleCase[0])
: new ScaleCase[] {loaded.get(Integer.parseInt(index))};
}

@TearDown(Level.Trial)
public void tearDown() {
for (final ScaleCase scaleCase : scaleCases) {
scaleCase.check(scaleCase.evaluate(provider));
}
provider.shutdown();
}

@Benchmark
public ProviderEvaluation<?> evaluate(final Cursor cursor) {
return cursor.next(scaleCases).evaluate(provider);
}

/** As {@link #evaluate}, with the fresh layered context that the OpenFeature client builds. */
@Benchmark
public ProviderEvaluation<?> evaluateLayered(final Cursor cursor) {
final ScaleCase scaleCase = cursor.next(scaleCases);
return scaleCase.evaluate(
provider,
new LayeredEvaluationContext(
ImmutableContext.EMPTY, ImmutableContext.EMPTY, null, scaleCase.context()));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package com.datadog.openfeature;

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;

import com.datadog.openfeature.internal.ufc.ServerConfiguration;
import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser;
import de.thetaphi.forbiddenapis.SuppressForbidden;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.ref.Reference;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;

/**
* Configuration load cost of the opt-in scale suites: the time to parse each suite's {@code
* ufc-config.json}. Work that moves into the parser, such as indexes, compiled patterns or prebuilt
* JSON values, shows here.
*
* <p>The setup also prints the heap that a parsed configuration retains. It is the used heap after
* a full collection, with and without the parsed configuration, as the smallest of three
* measurements, so it is approximate.
*
* <p>Run: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteLoadBenchmark
* -Pjmh.profilers=gc}.
*/
@State(Scope.Benchmark)
@Warmup(iterations = 3, time = 2, timeUnit = SECONDS)
@Measurement(iterations = 5, time = 2, timeUnit = SECONDS)
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(MILLISECONDS)
@Fork(value = 1)
public class ScaleSuiteLoadBenchmark {

@Param({"scale", "scale-json"})
public String suite;

private byte[] config;

@Setup(Level.Trial)
@SuppressForbidden
public void setUp() throws IOException {
config = ScaleSuites.load(suite).config();
// The first parse initializes the parser, so it is left out of the measurement.
ServerConfiguration retained = UniversalFlagConfigParser.parse(config);
long retainedBytes = Long.MAX_VALUE;
for (int i = 0; i < 3; i++) {
retained = null;
final long before = usedHeapAfterCollection();
retained = UniversalFlagConfigParser.parse(config);
final long after = usedHeapAfterCollection();
Reference.reachabilityFence(retained);
retainedBytes = Math.min(retainedBytes, after - before);
}
System.out.println(
"["
+ suite
+ "] config "
+ config.length / 1024
+ " KB, "
+ retained.flags.size()
+ " flags, retained heap about "
+ retainedBytes / 1024
+ " KB");
}

@Benchmark
public ServerConfiguration parse() throws IOException {
return UniversalFlagConfigParser.parse(config);
}

private static long usedHeapAfterCollection() {
final MemoryMXBean memory = ManagementFactory.getMemoryMXBean();
for (int i = 0; i < 3; i++) {
System.gc();
}
return memory.getHeapMemoryUsage().getUsed();
}
}
1 change: 1 addition & 0 deletions dd-openfeature/src/test/fixtures/ffe-system-test-data
Submodule ffe-system-test-data added at 508a34
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
*/
class CanonicalFixtureTest {
private static final String CANONICAL_FIXTURE_PATH =
"dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data";
"dd-openfeature/src/test/fixtures/ffe-system-test-data";

@Test
void canonicalFixturesArePresent() throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -483,7 +483,7 @@ private static Path fixture(final String name) {
while (directory != null) {
final Path candidate =
directory
.resolve("dd-smoke-tests/openfeature/src/test/resources/experimentation-opt-out")
.resolve("dd-openfeature/src/test/fixtures/experimentation-opt-out")
.resolve(name);
if (Files.isRegularFile(candidate)) {
return candidate;
Expand Down
Loading