From f5968b78e8840988e950c07ede9e150b6ab9844f Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:33:19 +0000 Subject: [PATCH 1/3] Stop taking the OpenTelemetry global lock per evaluation GlobalOpenTelemetry.getOrNoop() locks a global mutex on every call. Keep the counter of a registered SDK, which the global cannot replace, look for an SDK at most every 100 ms until one is registered, and skip the attributes while OpenTelemetry is the no-op one. Co-Authored-By: Claude Opus 5.5 (1M context) Co-authored-by: dougqh --- .../datadog/openfeature/FlagEvalMetrics.java | 60 ++++++++++++++++++- .../openfeature/FlagEvalMetricsHook.java | 6 +- .../FlagEvalMetricsLateSdkForkedTest.java | 2 +- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java index a108e0fd582..4521608551c 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java @@ -1,6 +1,7 @@ package com.datadog.openfeature; import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.Reason; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; @@ -9,6 +10,7 @@ import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.Meter; import java.io.Closeable; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,26 +33,68 @@ class FlagEvalMetrics implements Closeable { private static final AttributeKey ATTR_ALLOCATION_KEY = AttributeKey.stringKey("feature_flag.result.allocation_key"); + // The attribute values of the standard reasons and error codes, lower-cased once. + private static final String[] REASONS = new String[Reason.values().length]; + private static final String[] LOWER_CASE_REASONS = new String[REASONS.length]; + private static final String[] ERROR_TYPES = new String[ErrorCode.values().length]; + + static { + for (final Reason reason : Reason.values()) { + REASONS[reason.ordinal()] = reason.name(); + LOWER_CASE_REASONS[reason.ordinal()] = reason.name().toLowerCase(); + } + for (final ErrorCode errorCode : ErrorCode.values()) { + ERROR_TYPES[errorCode.ordinal()] = errorCode.name().toLowerCase(); + } + } + + /** How long the metrics wait for an SDK before looking for one again. */ + static final long NOOP_RECHECK_NANOS = TimeUnit.MILLISECONDS.toNanos(100); + private volatile LongCounter counter; private volatile OpenTelemetry telemetry; private volatile boolean closed; private volatile Boolean legacyAgentBridge; private volatile boolean getOrNoopUnavailable; private final boolean suppliedCounter; + private final long noopRecheckNanos; + private volatile long lastNoopCheckNanos; FlagEvalMetrics() { - suppliedCounter = false; + this(NOOP_RECHECK_NANOS); + } + + FlagEvalMetrics(final long noopRecheckNanos) { + this.suppliedCounter = false; + this.noopRecheckNanos = noopRecheckNanos; + this.lastNoopCheckNanos = System.nanoTime() - noopRecheckNanos; } private LongCounter resolveCounter() { if (closed || suppliedCounter) { return counter; } + // GlobalOpenTelemetry.getOrNoop() takes a global lock on every call, so it is not called per + // evaluation. The global can be set only once, so the counter of a registered SDK stays valid. + final LongCounter resolved = counter; + if (resolved != null) { + return resolved; + } + // Without an SDK, look again from time to time: one may still be registered. + final long now = System.nanoTime(); + if (now - lastNoopCheckNanos < noopRecheckNanos) { + return null; + } + lastNoopCheckNanos = now; try { final OpenTelemetry current = currentOpenTelemetry(); if (current == telemetry) { return counter; } + if (current == OpenTelemetry.noop()) { + // Nothing to record into until the application registers an SDK: skip the attributes. + return null; + } return initializeCounter(current); } catch (LinkageError e) { log.debug("OpenTelemetry API is not available for evaluation metrics", e); @@ -118,6 +162,7 @@ private synchronized LongCounter initializeCounter(final OpenTelemetry current) /** Package-private constructor for testing with a mock counter. */ FlagEvalMetrics(LongCounter counter) { this.suppliedCounter = true; + this.noopRecheckNanos = 0; this.counter = counter; } @@ -132,10 +177,10 @@ void record( Attributes.builder() .put(ATTR_FLAG_KEY, flagKey) .put(ATTR_VARIANT, variant != null ? variant : "") - .put(ATTR_REASON, reason != null ? reason.toLowerCase() : "unknown"); + .put(ATTR_REASON, reason != null ? lowerCaseReason(reason) : "unknown"); if (errorCode != null) { - builder.put(ATTR_ERROR_TYPE, errorCode.name().toLowerCase()); + builder.put(ATTR_ERROR_TYPE, ERROR_TYPES[errorCode.ordinal()]); } if (allocationKey != null && !allocationKey.isEmpty()) { @@ -148,6 +193,15 @@ void record( } } + private static String lowerCaseReason(final String reason) { + for (int i = 0; i < REASONS.length; i++) { + if (REASONS[i].equals(reason)) { + return LOWER_CASE_REASONS[i]; + } + } + return reason.toLowerCase(); + } + @Override public void close() { shutdown(); diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java index d63cbe56d8c..05214964fcc 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java @@ -1,5 +1,7 @@ package com.datadog.openfeature; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY; + import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.Hook; @@ -30,7 +32,9 @@ public void finallyAfter( String allocationKey = null; ImmutableMetadata metadata = details.getFlagMetadata(); if (metadata != null) { - allocationKey = metadata.getString("allocationKey"); + // The ImmutableMetadata getters build a debug message for every absent key. + final Object allocationValue = metadata.asUnmodifiableMap().get(ALLOCATION_KEY); + allocationKey = allocationValue instanceof String ? (String) allocationValue : null; } metrics.record(flagKey, variant, reason, errorCode, allocationKey); diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java index 80d212a10f8..0ffc377725e 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java @@ -22,7 +22,7 @@ class FlagEvalMetricsLateSdkForkedTest { @Test void evaluationBeforeSdkRegistrationDoesNotClaimTheGlobalProvider() { GlobalOpenTelemetry.resetForTest(); - final FlagEvalMetrics metrics = new FlagEvalMetrics(); + final FlagEvalMetrics metrics = new FlagEvalMetrics(0); metrics.record("before", "on", "STATIC", null, null); assertFalse(GlobalOpenTelemetry.isSet()); From f0678ebce4196ce7efbddb8ddf30b4180486504d Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 20 Nov 2026 07:35:22 -0500 Subject: [PATCH 2/3] Skip evaluation metrics only when the meter provider records nothing With dd-java-agent and DD_METRICS_OTEL_ENABLED, the agent makes the no-op OpenTelemetry instance return a meter provider that records, so skipping the no-op instance dropped feature_flag.evaluations entirely. Checking the meter provider instead also skips the wrapped no-op that older APIs return from GlobalOpenTelemetry.get(), which still built attributes per evaluation. Also drop the telemetry field the keep-forever counter made redundant, lower-case with Locale.ROOT, and read the allocation key through one MetadataKeys helper instead of four copies. Co-Authored-By: Claude Opus 5.5 --- .../com/datadog/openfeature/ExposureHook.java | 4 +-- .../openfeature/ExposureLoggingHook.java | 6 ++-- .../openfeature/FlagEvalLoggingHook.java | 6 ++-- .../datadog/openfeature/FlagEvalMetrics.java | 33 ++++++++++++------- .../openfeature/FlagEvalMetricsHook.java | 6 ++-- .../internal/evaluation/MetadataKeys.java | 14 ++++++++ .../openfeature/FlagEvalMetricsTest.java | 21 ++++++++++++ 7 files changed, 65 insertions(+), 25 deletions(-) diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureHook.java index 4aba33dc10e..1a04894aa96 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureHook.java @@ -4,6 +4,7 @@ import static com.datadog.openfeature.internal.evaluation.MetadataKeys.DO_LOG; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_CACHE_HIT; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.FEATURE_PREFIX; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; @@ -130,8 +131,7 @@ public String getVariant() { * when it failed or returned the default value. */ public String getAllocationKey() { - final Object allocationKey = metadata().get(ALLOCATION_KEY); - return allocationKey instanceof String ? (String) allocationKey : null; + return allocationKey(metadata()); } /** diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureLoggingHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureLoggingHook.java index db6eba8a70d..d66c83d61ac 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureLoggingHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureLoggingHook.java @@ -1,10 +1,10 @@ package com.datadog.openfeature; -import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.DO_LOG; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_CACHE_HIT; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_FEATURE_PREFIX; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.SPLIT_SERIAL_ID; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey; import com.datadog.openfeature.internal.exposure.ExposureDeduplicationCache; import dev.openfeature.sdk.EvaluationContext; @@ -65,9 +65,7 @@ public void finallyAfter( log.trace("Feature Flagging exposure suppressed: exposure already recorded"); return; } - final Object allocationValue = values.get(ALLOCATION_KEY); - final String allocationKey = - allocationValue instanceof String ? (String) allocationValue : null; + final String allocationKey = allocationKey(values); final String variantKey = details.getVariant(); final EvaluationContext context = ctx == null ? null : ctx.getCtx(); if (allocationKey == null || variantKey == null || context == null) { diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalLoggingHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalLoggingHook.java index 504ce4a3337..bbbfa467914 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalLoggingHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalLoggingHook.java @@ -1,10 +1,10 @@ package com.datadog.openfeature; -import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVALUATION_FEATURE_PREFIX; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVAL_TIMESTAMP_MS; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.HAS_EVALUATION_FEATURES; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.OBSERVE_FULL_EVALUATION_DATA; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey; import com.datadog.openfeature.internal.flagevaluation.FlagEvalEvent; import com.datadog.openfeature.internal.flagevaluation.FlagEvaluationWriter; @@ -82,9 +82,7 @@ public void finallyAfter( // The ImmutableMetadata getters build a debug message for every absent key. final Map values = metadata == null ? Collections.emptyMap() : metadata.asUnmodifiableMap(); - final Object allocationValue = values.get(ALLOCATION_KEY); - final String allocationKey = - allocationValue instanceof String ? (String) allocationValue : null; + final String allocationKey = allocationKey(values); // eval-time: from flag metadata "__dd_eval_timestamp_ms" (Long), fallback to hook-fire time. final Object evalTimeValue = values.get(EVAL_TIMESTAMP_MS); diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java index 4521608551c..e7b525863a1 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java @@ -9,7 +9,9 @@ import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; import java.io.Closeable; +import java.util.Locale; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,10 +43,10 @@ class FlagEvalMetrics implements Closeable { static { for (final Reason reason : Reason.values()) { REASONS[reason.ordinal()] = reason.name(); - LOWER_CASE_REASONS[reason.ordinal()] = reason.name().toLowerCase(); + LOWER_CASE_REASONS[reason.ordinal()] = reason.name().toLowerCase(Locale.ROOT); } for (final ErrorCode errorCode : ErrorCode.values()) { - ERROR_TYPES[errorCode.ordinal()] = errorCode.name().toLowerCase(); + ERROR_TYPES[errorCode.ordinal()] = errorCode.name().toLowerCase(Locale.ROOT); } } @@ -52,7 +54,6 @@ class FlagEvalMetrics implements Closeable { static final long NOOP_RECHECK_NANOS = TimeUnit.MILLISECONDS.toNanos(100); private volatile LongCounter counter; - private volatile OpenTelemetry telemetry; private volatile boolean closed; private volatile Boolean legacyAgentBridge; private volatile boolean getOrNoopUnavailable; @@ -64,6 +65,11 @@ class FlagEvalMetrics implements Closeable { this(NOOP_RECHECK_NANOS); } + /** + * Creates metrics that look for an SDK again at most once per interval while none is registered. + * + * @param noopRecheckNanos the interval, in nanoseconds; tests pass 0 to look on every evaluation. + */ FlagEvalMetrics(final long noopRecheckNanos) { this.suppliedCounter = false; this.noopRecheckNanos = noopRecheckNanos; @@ -88,10 +94,7 @@ private LongCounter resolveCounter() { lastNoopCheckNanos = now; try { final OpenTelemetry current = currentOpenTelemetry(); - if (current == telemetry) { - return counter; - } - if (current == OpenTelemetry.noop()) { + if (recordsNothing(current)) { // Nothing to record into until the application registers an SDK: skip the attributes. return null; } @@ -105,6 +108,15 @@ private LongCounter resolveCounter() { return null; } + /** + * Tells whether an OpenTelemetry instance drops every metric. Checked on its meter provider, not + * on the instance: dd-java-agent's OpenTelemetry metrics support makes the no-op instance's meter + * provider record, and older APIs wrap the no-op instance in another one. + */ + static boolean recordsNothing(final OpenTelemetry telemetry) { + return telemetry.getMeterProvider() == MeterProvider.noop(); + } + private OpenTelemetry currentOpenTelemetry() { if (!getOrNoopUnavailable) { try { @@ -144,7 +156,8 @@ private synchronized LongCounter initializeCounter(final OpenTelemetry current) if (closed) { return null; } - if (current != telemetry) { + // Another thread may have created the counter while this one waited for the lock. + if (counter == null) { Meter meter = current.getMeterProvider().meterBuilder(METER_NAME).build(); counter = meter @@ -152,7 +165,6 @@ private synchronized LongCounter initializeCounter(final OpenTelemetry current) .setUnit(METRIC_UNIT) .setDescription(METRIC_DESC) .build(); - telemetry = current; log.debug("Flag evaluation metrics initialized"); } @@ -199,7 +211,7 @@ private static String lowerCaseReason(final String reason) { return LOWER_CASE_REASONS[i]; } } - return reason.toLowerCase(); + return reason.toLowerCase(Locale.ROOT); } @Override @@ -210,6 +222,5 @@ public void close() { synchronized void shutdown() { closed = true; counter = null; - telemetry = null; } } diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java index 05214964fcc..40e22204720 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java @@ -1,6 +1,6 @@ package com.datadog.openfeature; -import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey; import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.FlagEvaluationDetails; @@ -32,9 +32,7 @@ public void finallyAfter( String allocationKey = null; ImmutableMetadata metadata = details.getFlagMetadata(); if (metadata != null) { - // The ImmutableMetadata getters build a debug message for every absent key. - final Object allocationValue = metadata.asUnmodifiableMap().get(ALLOCATION_KEY); - allocationKey = allocationValue instanceof String ? (String) allocationValue : null; + allocationKey = allocationKey(metadata.asUnmodifiableMap()); } metrics.record(flagKey, variant, reason, errorCode, allocationKey); diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/MetadataKeys.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/MetadataKeys.java index d7a19ac79cf..caefa2e2d82 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/MetadataKeys.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/MetadataKeys.java @@ -1,10 +1,24 @@ package com.datadog.openfeature.internal.evaluation; +import java.util.Map; + /** The flag metadata keys the evaluator writes for the provider hooks and exposure hooks. */ public final class MetadataKeys { /** The key of the selected allocation. */ public static final String ALLOCATION_KEY = "allocationKey"; + /** + * Reads the allocation key from flag metadata. Read from the map because the {@code + * ImmutableMetadata} getters build a debug message for every absent key. + * + * @param values the flag metadata, as {@code ImmutableMetadata.asUnmodifiableMap()}. + * @return the allocation key, or {@code null} when absent or not a string. + */ + public static String allocationKey(final Map values) { + final Object value = values.get(ALLOCATION_KEY); + return value instanceof String ? (String) value : null; + } + /** The evaluation timestamp, in milliseconds since epoch. */ public static final String EVAL_TIMESTAMP_MS = "__dd_eval_timestamp_ms"; diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsTest.java index 61e0ab4dd4b..7ec521dc59b 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsTest.java @@ -1,13 +1,18 @@ package com.datadog.openfeature; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; import dev.openfeature.sdk.ErrorCode; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.MeterProvider; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -162,4 +167,20 @@ private static void assertNoAttribute(Attributes attrs, String key) { throw new AssertionError("Expected no attribute " + key + " but it was present"); } } + + @Test + void skipsOnlyAnOpenTelemetryWhoseMeterProviderRecordsNothing() { + assertTrue(FlagEvalMetrics.recordsNothing(OpenTelemetry.noop())); + + // Older APIs wrap the no-op instance in another one with the same no-op meter provider. + final OpenTelemetry wrappedNoop = mock(OpenTelemetry.class); + when(wrappedNoop.getMeterProvider()).thenReturn(MeterProvider.noop()); + assertTrue(FlagEvalMetrics.recordsNothing(wrappedNoop)); + + // dd-java-agent with DD_METRICS_OTEL_ENABLED makes the no-op instance return its own meter + // provider, which records: the evaluations must still be counted there. + final OpenTelemetry agentRewritten = mock(OpenTelemetry.class); + when(agentRewritten.getMeterProvider()).thenReturn(mock(MeterProvider.class)); + assertFalse(FlagEvalMetrics.recordsNothing(agentRewritten)); + } } From 209f4e6947ca7b77a7278d2cc80b9d94370d96c3 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 20 Nov 2026 16:36:08 -0500 Subject: [PATCH 3/3] Throttle the metrics SDK lookup only after seeing a no-op The recheck timestamp was set before the counter was built, so an evaluation racing the first one's counter initialization returned early and went unrecorded. It's now set only when the lookup finds a no-op provider, so racing evaluations wait in initializeCounter and record. From Codex's review. Co-Authored-By: Claude Opus 5.5 --- .../datadog/openfeature/FlagEvalMetrics.java | 4 +- .../FlagEvalMetricsLateSdkForkedTest.java | 56 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java index e7b525863a1..dbd81992cba 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java @@ -91,11 +91,13 @@ private LongCounter resolveCounter() { if (now - lastNoopCheckNanos < noopRecheckNanos) { return null; } - lastNoopCheckNanos = now; try { final OpenTelemetry current = currentOpenTelemetry(); if (recordsNothing(current)) { // Nothing to record into until the application registers an SDK: skip the attributes. + // Throttled only once a no-op is seen, so evaluations racing a real SDK's counter + // initialization wait for it in initializeCounter rather than going unrecorded. + lastNoopCheckNanos = now; return null; } return initializeCounter(current); diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java index 0ffc377725e..42dd78a4530 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java @@ -2,9 +2,11 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -16,6 +18,8 @@ import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.MeterBuilder; import io.opentelemetry.api.metrics.MeterProvider; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; class FlagEvalMetricsLateSdkForkedTest { @@ -49,4 +53,56 @@ void evaluationBeforeSdkRegistrationDoesNotClaimTheGlobalProvider() { GlobalOpenTelemetry.resetForTest(); } } + + @Test + void evaluationsRacingCounterInitializationAreStillRecorded() throws Exception { + GlobalOpenTelemetry.resetForTest(); + final CountDownLatch building = new CountDownLatch(1); + final CountDownLatch release = new CountDownLatch(1); + final OpenTelemetry telemetry = mock(OpenTelemetry.class); + final MeterProvider provider = mock(MeterProvider.class); + final MeterBuilder meterBuilder = mock(MeterBuilder.class); + final Meter meter = mock(Meter.class); + final LongCounterBuilder counterBuilder = mock(LongCounterBuilder.class); + final LongCounter counter = mock(LongCounter.class); + when(telemetry.getMeterProvider()).thenReturn(provider); + when(provider.meterBuilder("ddtrace.openfeature")) + .thenAnswer(invocation -> { + building.countDown(); + release.await(); // hold the first evaluation inside counter initialization + return meterBuilder; + }); + when(meterBuilder.build()).thenReturn(meter); + when(meter.counterBuilder("feature_flag.evaluations")).thenReturn(counterBuilder); + when(counterBuilder.setUnit("{evaluation}")).thenReturn(counterBuilder); + when(counterBuilder.setDescription("Number of feature flag evaluations")) + .thenReturn(counterBuilder); + when(counterBuilder.build()).thenReturn(counter); + GlobalOpenTelemetry.set(telemetry); + // A long recheck interval: a throttled evaluation would be dropped, not retried. + final FlagEvalMetrics metrics = new FlagEvalMetrics(TimeUnit.DAYS.toNanos(1)); + try { + final Thread first = new Thread(() -> metrics.record("first", "on", "STATIC", null, null)); + first.start(); + assertTrue(building.await(10, TimeUnit.SECONDS)); + final Thread second = new Thread(() -> metrics.record("second", "on", "STATIC", null, null)); + second.start(); + // Wait until the second evaluation is blocked behind the first's initialization. + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (second.getState() != Thread.State.BLOCKED && second.isAlive()) { + if (System.nanoTime() > deadline) { + throw new AssertionError("second evaluation never reached the initializer"); + } + Thread.sleep(1); + } + release.countDown(); + first.join(10_000); + second.join(10_000); + verify(counter, times(2)).add(eq(1L), any(Attributes.class)); + } finally { + release.countDown(); + metrics.close(); + GlobalOpenTelemetry.resetForTest(); + } + } }