diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureHook.java index 4aba33dc10e..1a04894aa96 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureHook.java @@ -4,6 +4,7 @@ import static com.datadog.openfeature.internal.evaluation.MetadataKeys.DO_LOG; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_CACHE_HIT; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.FEATURE_PREFIX; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; @@ -130,8 +131,7 @@ public String getVariant() { * when it failed or returned the default value. */ public String getAllocationKey() { - final Object allocationKey = metadata().get(ALLOCATION_KEY); - return allocationKey instanceof String ? (String) allocationKey : null; + return allocationKey(metadata()); } /** diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureLoggingHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureLoggingHook.java index db6eba8a70d..d66c83d61ac 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureLoggingHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/ExposureLoggingHook.java @@ -1,10 +1,10 @@ package com.datadog.openfeature; -import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.DO_LOG; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_CACHE_HIT; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_FEATURE_PREFIX; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.SPLIT_SERIAL_ID; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey; import com.datadog.openfeature.internal.exposure.ExposureDeduplicationCache; import dev.openfeature.sdk.EvaluationContext; @@ -65,9 +65,7 @@ public void finallyAfter( log.trace("Feature Flagging exposure suppressed: exposure already recorded"); return; } - final Object allocationValue = values.get(ALLOCATION_KEY); - final String allocationKey = - allocationValue instanceof String ? (String) allocationValue : null; + final String allocationKey = allocationKey(values); final String variantKey = details.getVariant(); final EvaluationContext context = ctx == null ? null : ctx.getCtx(); if (allocationKey == null || variantKey == null || context == null) { diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalLoggingHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalLoggingHook.java index 504ce4a3337..bbbfa467914 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalLoggingHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalLoggingHook.java @@ -1,10 +1,10 @@ package com.datadog.openfeature; -import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVALUATION_FEATURE_PREFIX; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVAL_TIMESTAMP_MS; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.HAS_EVALUATION_FEATURES; import static com.datadog.openfeature.internal.evaluation.MetadataKeys.OBSERVE_FULL_EVALUATION_DATA; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey; import com.datadog.openfeature.internal.flagevaluation.FlagEvalEvent; import com.datadog.openfeature.internal.flagevaluation.FlagEvaluationWriter; @@ -82,9 +82,7 @@ public void finallyAfter( // The ImmutableMetadata getters build a debug message for every absent key. final Map values = metadata == null ? Collections.emptyMap() : metadata.asUnmodifiableMap(); - final Object allocationValue = values.get(ALLOCATION_KEY); - final String allocationKey = - allocationValue instanceof String ? (String) allocationValue : null; + final String allocationKey = allocationKey(values); // eval-time: from flag metadata "__dd_eval_timestamp_ms" (Long), fallback to hook-fire time. final Object evalTimeValue = values.get(EVAL_TIMESTAMP_MS); diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java index a108e0fd582..dbd81992cba 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetrics.java @@ -1,6 +1,7 @@ package com.datadog.openfeature; import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.Reason; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; @@ -8,7 +9,10 @@ import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; import java.io.Closeable; +import java.util.Locale; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,25 +35,70 @@ class FlagEvalMetrics implements Closeable { private static final AttributeKey ATTR_ALLOCATION_KEY = AttributeKey.stringKey("feature_flag.result.allocation_key"); + // The attribute values of the standard reasons and error codes, lower-cased once. + private static final String[] REASONS = new String[Reason.values().length]; + private static final String[] LOWER_CASE_REASONS = new String[REASONS.length]; + private static final String[] ERROR_TYPES = new String[ErrorCode.values().length]; + + static { + for (final Reason reason : Reason.values()) { + REASONS[reason.ordinal()] = reason.name(); + LOWER_CASE_REASONS[reason.ordinal()] = reason.name().toLowerCase(Locale.ROOT); + } + for (final ErrorCode errorCode : ErrorCode.values()) { + ERROR_TYPES[errorCode.ordinal()] = errorCode.name().toLowerCase(Locale.ROOT); + } + } + + /** How long the metrics wait for an SDK before looking for one again. */ + static final long NOOP_RECHECK_NANOS = TimeUnit.MILLISECONDS.toNanos(100); + private volatile LongCounter counter; - private volatile OpenTelemetry telemetry; private volatile boolean closed; private volatile Boolean legacyAgentBridge; private volatile boolean getOrNoopUnavailable; private final boolean suppliedCounter; + private final long noopRecheckNanos; + private volatile long lastNoopCheckNanos; FlagEvalMetrics() { - suppliedCounter = false; + this(NOOP_RECHECK_NANOS); + } + + /** + * Creates metrics that look for an SDK again at most once per interval while none is registered. + * + * @param noopRecheckNanos the interval, in nanoseconds; tests pass 0 to look on every evaluation. + */ + FlagEvalMetrics(final long noopRecheckNanos) { + this.suppliedCounter = false; + this.noopRecheckNanos = noopRecheckNanos; + this.lastNoopCheckNanos = System.nanoTime() - noopRecheckNanos; } private LongCounter resolveCounter() { if (closed || suppliedCounter) { return counter; } + // GlobalOpenTelemetry.getOrNoop() takes a global lock on every call, so it is not called per + // evaluation. The global can be set only once, so the counter of a registered SDK stays valid. + final LongCounter resolved = counter; + if (resolved != null) { + return resolved; + } + // Without an SDK, look again from time to time: one may still be registered. + final long now = System.nanoTime(); + if (now - lastNoopCheckNanos < noopRecheckNanos) { + return null; + } try { final OpenTelemetry current = currentOpenTelemetry(); - if (current == telemetry) { - return counter; + if (recordsNothing(current)) { + // Nothing to record into until the application registers an SDK: skip the attributes. + // Throttled only once a no-op is seen, so evaluations racing a real SDK's counter + // initialization wait for it in initializeCounter rather than going unrecorded. + lastNoopCheckNanos = now; + return null; } return initializeCounter(current); } catch (LinkageError e) { @@ -61,6 +110,15 @@ private LongCounter resolveCounter() { return null; } + /** + * Tells whether an OpenTelemetry instance drops every metric. Checked on its meter provider, not + * on the instance: dd-java-agent's OpenTelemetry metrics support makes the no-op instance's meter + * provider record, and older APIs wrap the no-op instance in another one. + */ + static boolean recordsNothing(final OpenTelemetry telemetry) { + return telemetry.getMeterProvider() == MeterProvider.noop(); + } + private OpenTelemetry currentOpenTelemetry() { if (!getOrNoopUnavailable) { try { @@ -100,7 +158,8 @@ private synchronized LongCounter initializeCounter(final OpenTelemetry current) if (closed) { return null; } - if (current != telemetry) { + // Another thread may have created the counter while this one waited for the lock. + if (counter == null) { Meter meter = current.getMeterProvider().meterBuilder(METER_NAME).build(); counter = meter @@ -108,7 +167,6 @@ private synchronized LongCounter initializeCounter(final OpenTelemetry current) .setUnit(METRIC_UNIT) .setDescription(METRIC_DESC) .build(); - telemetry = current; log.debug("Flag evaluation metrics initialized"); } @@ -118,6 +176,7 @@ private synchronized LongCounter initializeCounter(final OpenTelemetry current) /** Package-private constructor for testing with a mock counter. */ FlagEvalMetrics(LongCounter counter) { this.suppliedCounter = true; + this.noopRecheckNanos = 0; this.counter = counter; } @@ -132,10 +191,10 @@ void record( Attributes.builder() .put(ATTR_FLAG_KEY, flagKey) .put(ATTR_VARIANT, variant != null ? variant : "") - .put(ATTR_REASON, reason != null ? reason.toLowerCase() : "unknown"); + .put(ATTR_REASON, reason != null ? lowerCaseReason(reason) : "unknown"); if (errorCode != null) { - builder.put(ATTR_ERROR_TYPE, errorCode.name().toLowerCase()); + builder.put(ATTR_ERROR_TYPE, ERROR_TYPES[errorCode.ordinal()]); } if (allocationKey != null && !allocationKey.isEmpty()) { @@ -148,6 +207,15 @@ void record( } } + private static String lowerCaseReason(final String reason) { + for (int i = 0; i < REASONS.length; i++) { + if (REASONS[i].equals(reason)) { + return LOWER_CASE_REASONS[i]; + } + } + return reason.toLowerCase(Locale.ROOT); + } + @Override public void close() { shutdown(); @@ -156,6 +224,5 @@ public void close() { synchronized void shutdown() { closed = true; counter = null; - telemetry = null; } } diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java index d63cbe56d8c..40e22204720 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/FlagEvalMetricsHook.java @@ -1,5 +1,7 @@ package com.datadog.openfeature; +import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey; + import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.Hook; @@ -30,7 +32,7 @@ public void finallyAfter( String allocationKey = null; ImmutableMetadata metadata = details.getFlagMetadata(); if (metadata != null) { - allocationKey = metadata.getString("allocationKey"); + allocationKey = allocationKey(metadata.asUnmodifiableMap()); } metrics.record(flagKey, variant, reason, errorCode, allocationKey); diff --git a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/MetadataKeys.java b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/MetadataKeys.java index d7a19ac79cf..caefa2e2d82 100644 --- a/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/MetadataKeys.java +++ b/dd-openfeature/src/main/java/com/datadog/openfeature/internal/evaluation/MetadataKeys.java @@ -1,10 +1,24 @@ package com.datadog.openfeature.internal.evaluation; +import java.util.Map; + /** The flag metadata keys the evaluator writes for the provider hooks and exposure hooks. */ public final class MetadataKeys { /** The key of the selected allocation. */ public static final String ALLOCATION_KEY = "allocationKey"; + /** + * Reads the allocation key from flag metadata. Read from the map because the {@code + * ImmutableMetadata} getters build a debug message for every absent key. + * + * @param values the flag metadata, as {@code ImmutableMetadata.asUnmodifiableMap()}. + * @return the allocation key, or {@code null} when absent or not a string. + */ + public static String allocationKey(final Map values) { + final Object value = values.get(ALLOCATION_KEY); + return value instanceof String ? (String) value : null; + } + /** The evaluation timestamp, in milliseconds since epoch. */ public static final String EVAL_TIMESTAMP_MS = "__dd_eval_timestamp_ms"; diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java index 80d212a10f8..42dd78a4530 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsLateSdkForkedTest.java @@ -2,9 +2,11 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -16,13 +18,15 @@ import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.MeterBuilder; import io.opentelemetry.api.metrics.MeterProvider; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; class FlagEvalMetricsLateSdkForkedTest { @Test void evaluationBeforeSdkRegistrationDoesNotClaimTheGlobalProvider() { GlobalOpenTelemetry.resetForTest(); - final FlagEvalMetrics metrics = new FlagEvalMetrics(); + final FlagEvalMetrics metrics = new FlagEvalMetrics(0); metrics.record("before", "on", "STATIC", null, null); assertFalse(GlobalOpenTelemetry.isSet()); @@ -49,4 +53,56 @@ void evaluationBeforeSdkRegistrationDoesNotClaimTheGlobalProvider() { GlobalOpenTelemetry.resetForTest(); } } + + @Test + void evaluationsRacingCounterInitializationAreStillRecorded() throws Exception { + GlobalOpenTelemetry.resetForTest(); + final CountDownLatch building = new CountDownLatch(1); + final CountDownLatch release = new CountDownLatch(1); + final OpenTelemetry telemetry = mock(OpenTelemetry.class); + final MeterProvider provider = mock(MeterProvider.class); + final MeterBuilder meterBuilder = mock(MeterBuilder.class); + final Meter meter = mock(Meter.class); + final LongCounterBuilder counterBuilder = mock(LongCounterBuilder.class); + final LongCounter counter = mock(LongCounter.class); + when(telemetry.getMeterProvider()).thenReturn(provider); + when(provider.meterBuilder("ddtrace.openfeature")) + .thenAnswer(invocation -> { + building.countDown(); + release.await(); // hold the first evaluation inside counter initialization + return meterBuilder; + }); + when(meterBuilder.build()).thenReturn(meter); + when(meter.counterBuilder("feature_flag.evaluations")).thenReturn(counterBuilder); + when(counterBuilder.setUnit("{evaluation}")).thenReturn(counterBuilder); + when(counterBuilder.setDescription("Number of feature flag evaluations")) + .thenReturn(counterBuilder); + when(counterBuilder.build()).thenReturn(counter); + GlobalOpenTelemetry.set(telemetry); + // A long recheck interval: a throttled evaluation would be dropped, not retried. + final FlagEvalMetrics metrics = new FlagEvalMetrics(TimeUnit.DAYS.toNanos(1)); + try { + final Thread first = new Thread(() -> metrics.record("first", "on", "STATIC", null, null)); + first.start(); + assertTrue(building.await(10, TimeUnit.SECONDS)); + final Thread second = new Thread(() -> metrics.record("second", "on", "STATIC", null, null)); + second.start(); + // Wait until the second evaluation is blocked behind the first's initialization. + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (second.getState() != Thread.State.BLOCKED && second.isAlive()) { + if (System.nanoTime() > deadline) { + throw new AssertionError("second evaluation never reached the initializer"); + } + Thread.sleep(1); + } + release.countDown(); + first.join(10_000); + second.join(10_000); + verify(counter, times(2)).add(eq(1L), any(Attributes.class)); + } finally { + release.countDown(); + metrics.close(); + GlobalOpenTelemetry.resetForTest(); + } + } } diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsTest.java index 61e0ab4dd4b..7ec521dc59b 100644 --- a/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsTest.java +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/FlagEvalMetricsTest.java @@ -1,13 +1,18 @@ package com.datadog.openfeature; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; import dev.openfeature.sdk.ErrorCode; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.MeterProvider; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -162,4 +167,20 @@ private static void assertNoAttribute(Attributes attrs, String key) { throw new AssertionError("Expected no attribute " + key + " but it was present"); } } + + @Test + void skipsOnlyAnOpenTelemetryWhoseMeterProviderRecordsNothing() { + assertTrue(FlagEvalMetrics.recordsNothing(OpenTelemetry.noop())); + + // Older APIs wrap the no-op instance in another one with the same no-op meter provider. + final OpenTelemetry wrappedNoop = mock(OpenTelemetry.class); + when(wrappedNoop.getMeterProvider()).thenReturn(MeterProvider.noop()); + assertTrue(FlagEvalMetrics.recordsNothing(wrappedNoop)); + + // dd-java-agent with DD_METRICS_OTEL_ENABLED makes the no-op instance return its own meter + // provider, which records: the evaluations must still be counted there. + final OpenTelemetry agentRewritten = mock(OpenTelemetry.class); + when(agentRewritten.getMeterProvider()).thenReturn(mock(MeterProvider.class)); + assertFalse(FlagEvalMetrics.recordsNothing(agentRewritten)); + } }