Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.DO_LOG;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_CACHE_HIT;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.FEATURE_PREFIX;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey;
import static java.util.Collections.emptyMap;
import static java.util.Objects.requireNonNull;

Expand Down Expand Up @@ -130,8 +131,7 @@ public String getVariant() {
* when it failed or returned the default value.
*/
public String getAllocationKey() {
final Object allocationKey = metadata().get(ALLOCATION_KEY);
return allocationKey instanceof String ? (String) allocationKey : null;
return allocationKey(metadata());
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package com.datadog.openfeature;

import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.DO_LOG;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_CACHE_HIT;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EXPOSURE_FEATURE_PREFIX;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.SPLIT_SERIAL_ID;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey;

import com.datadog.openfeature.internal.exposure.ExposureDeduplicationCache;
import dev.openfeature.sdk.EvaluationContext;
Expand Down Expand Up @@ -65,9 +65,7 @@ public void finallyAfter(
log.trace("Feature Flagging exposure suppressed: exposure already recorded");
return;
}
final Object allocationValue = values.get(ALLOCATION_KEY);
final String allocationKey =
allocationValue instanceof String ? (String) allocationValue : null;
final String allocationKey = allocationKey(values);
final String variantKey = details.getVariant();
final EvaluationContext context = ctx == null ? null : ctx.getCtx();
if (allocationKey == null || variantKey == null || context == null) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package com.datadog.openfeature;

import static com.datadog.openfeature.internal.evaluation.MetadataKeys.ALLOCATION_KEY;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVALUATION_FEATURE_PREFIX;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.EVAL_TIMESTAMP_MS;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.HAS_EVALUATION_FEATURES;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.OBSERVE_FULL_EVALUATION_DATA;
import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey;

import com.datadog.openfeature.internal.flagevaluation.FlagEvalEvent;
import com.datadog.openfeature.internal.flagevaluation.FlagEvaluationWriter;
Expand Down Expand Up @@ -82,9 +82,7 @@ public void finallyAfter(
// The ImmutableMetadata getters build a debug message for every absent key.
final Map<String, Object> values =
metadata == null ? Collections.<String, Object>emptyMap() : metadata.asUnmodifiableMap();
final Object allocationValue = values.get(ALLOCATION_KEY);
final String allocationKey =
allocationValue instanceof String ? (String) allocationValue : null;
final String allocationKey = allocationKey(values);

// eval-time: from flag metadata "__dd_eval_timestamp_ms" (Long), fallback to hook-fire time.
final Object evalTimeValue = values.get(EVAL_TIMESTAMP_MS);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
package com.datadog.openfeature;

import dev.openfeature.sdk.ErrorCode;
import dev.openfeature.sdk.Reason;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.common.AttributesBuilder;
import io.opentelemetry.api.metrics.LongCounter;
import io.opentelemetry.api.metrics.Meter;
import io.opentelemetry.api.metrics.MeterProvider;
import java.io.Closeable;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -31,25 +35,70 @@ class FlagEvalMetrics implements Closeable {
private static final AttributeKey<String> ATTR_ALLOCATION_KEY =
AttributeKey.stringKey("feature_flag.result.allocation_key");

// The attribute values of the standard reasons and error codes, lower-cased once.
private static final String[] REASONS = new String[Reason.values().length];
private static final String[] LOWER_CASE_REASONS = new String[REASONS.length];
private static final String[] ERROR_TYPES = new String[ErrorCode.values().length];

static {
for (final Reason reason : Reason.values()) {
REASONS[reason.ordinal()] = reason.name();
LOWER_CASE_REASONS[reason.ordinal()] = reason.name().toLowerCase(Locale.ROOT);
}
for (final ErrorCode errorCode : ErrorCode.values()) {
ERROR_TYPES[errorCode.ordinal()] = errorCode.name().toLowerCase(Locale.ROOT);
}
}

/** How long the metrics wait for an SDK before looking for one again. */
static final long NOOP_RECHECK_NANOS = TimeUnit.MILLISECONDS.toNanos(100);

private volatile LongCounter counter;
private volatile OpenTelemetry telemetry;
private volatile boolean closed;
private volatile Boolean legacyAgentBridge;
private volatile boolean getOrNoopUnavailable;
private final boolean suppliedCounter;
private final long noopRecheckNanos;
private volatile long lastNoopCheckNanos;

FlagEvalMetrics() {
suppliedCounter = false;
this(NOOP_RECHECK_NANOS);
}

/**
* Creates metrics that look for an SDK again at most once per interval while none is registered.
*
* @param noopRecheckNanos the interval, in nanoseconds; tests pass 0 to look on every evaluation.
*/
FlagEvalMetrics(final long noopRecheckNanos) {
this.suppliedCounter = false;
this.noopRecheckNanos = noopRecheckNanos;
this.lastNoopCheckNanos = System.nanoTime() - noopRecheckNanos;
}

private LongCounter resolveCounter() {
if (closed || suppliedCounter) {
return counter;
}
// GlobalOpenTelemetry.getOrNoop() takes a global lock on every call, so it is not called per
// evaluation. The global can be set only once, so the counter of a registered SDK stays valid.
final LongCounter resolved = counter;
if (resolved != null) {
return resolved;
}
// Without an SDK, look again from time to time: one may still be registered.
final long now = System.nanoTime();
if (now - lastNoopCheckNanos < noopRecheckNanos) {
return null;
}
try {
final OpenTelemetry current = currentOpenTelemetry();
if (current == telemetry) {
return counter;
if (recordsNothing(current)) {
// Nothing to record into until the application registers an SDK: skip the attributes.
// Throttled only once a no-op is seen, so evaluations racing a real SDK's counter
// initialization wait for it in initializeCounter rather than going unrecorded.
lastNoopCheckNanos = now;
return null;
}
return initializeCounter(current);
} catch (LinkageError e) {
Expand All @@ -61,6 +110,15 @@ private LongCounter resolveCounter() {
return null;
}

/**
* Tells whether an OpenTelemetry instance drops every metric. Checked on its meter provider, not
* on the instance: dd-java-agent's OpenTelemetry metrics support makes the no-op instance's meter
* provider record, and older APIs wrap the no-op instance in another one.
*/
static boolean recordsNothing(final OpenTelemetry telemetry) {
return telemetry.getMeterProvider() == MeterProvider.noop();
}

private OpenTelemetry currentOpenTelemetry() {
if (!getOrNoopUnavailable) {
try {
Expand Down Expand Up @@ -100,15 +158,15 @@ private synchronized LongCounter initializeCounter(final OpenTelemetry current)
if (closed) {
return null;
}
if (current != telemetry) {
// Another thread may have created the counter while this one waited for the lock.
if (counter == null) {
Meter meter = current.getMeterProvider().meterBuilder(METER_NAME).build();
counter =
meter
.counterBuilder(METRIC_NAME)
.setUnit(METRIC_UNIT)
.setDescription(METRIC_DESC)
.build();
telemetry = current;

log.debug("Flag evaluation metrics initialized");
}
Expand All @@ -118,6 +176,7 @@ private synchronized LongCounter initializeCounter(final OpenTelemetry current)
/** Package-private constructor for testing with a mock counter. */
FlagEvalMetrics(LongCounter counter) {
this.suppliedCounter = true;
this.noopRecheckNanos = 0;
this.counter = counter;
}

Expand All @@ -132,10 +191,10 @@ void record(
Attributes.builder()
.put(ATTR_FLAG_KEY, flagKey)
.put(ATTR_VARIANT, variant != null ? variant : "")
.put(ATTR_REASON, reason != null ? reason.toLowerCase() : "unknown");
.put(ATTR_REASON, reason != null ? lowerCaseReason(reason) : "unknown");

if (errorCode != null) {
builder.put(ATTR_ERROR_TYPE, errorCode.name().toLowerCase());
builder.put(ATTR_ERROR_TYPE, ERROR_TYPES[errorCode.ordinal()]);
}

if (allocationKey != null && !allocationKey.isEmpty()) {
Expand All @@ -148,6 +207,15 @@ void record(
}
}

private static String lowerCaseReason(final String reason) {
for (int i = 0; i < REASONS.length; i++) {
if (REASONS[i].equals(reason)) {
return LOWER_CASE_REASONS[i];
}
}
return reason.toLowerCase(Locale.ROOT);
}

@Override
public void close() {
shutdown();
Expand All @@ -156,6 +224,5 @@ public void close() {
synchronized void shutdown() {
closed = true;
counter = null;
telemetry = null;
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.datadog.openfeature;

import static com.datadog.openfeature.internal.evaluation.MetadataKeys.allocationKey;

import dev.openfeature.sdk.ErrorCode;
import dev.openfeature.sdk.FlagEvaluationDetails;
import dev.openfeature.sdk.Hook;
Expand Down Expand Up @@ -30,7 +32,7 @@ public void finallyAfter(
String allocationKey = null;
ImmutableMetadata metadata = details.getFlagMetadata();
if (metadata != null) {
allocationKey = metadata.getString("allocationKey");
allocationKey = allocationKey(metadata.asUnmodifiableMap());
}

metrics.record(flagKey, variant, reason, errorCode, allocationKey);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,24 @@
package com.datadog.openfeature.internal.evaluation;

import java.util.Map;

/** The flag metadata keys the evaluator writes for the provider hooks and exposure hooks. */
public final class MetadataKeys {
/** The key of the selected allocation. */
public static final String ALLOCATION_KEY = "allocationKey";

/**
* Reads the allocation key from flag metadata. Read from the map because the {@code
* ImmutableMetadata} getters build a debug message for every absent key.
*
* @param values the flag metadata, as {@code ImmutableMetadata.asUnmodifiableMap()}.
* @return the allocation key, or {@code null} when absent or not a string.
*/
public static String allocationKey(final Map<String, Object> values) {
final Object value = values.get(ALLOCATION_KEY);
return value instanceof String ? (String) value : null;
}

/** The evaluation timestamp, in milliseconds since epoch. */
public static final String EVAL_TIMESTAMP_MS = "__dd_eval_timestamp_ms";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

Expand All @@ -16,13 +18,15 @@
import io.opentelemetry.api.metrics.Meter;
import io.opentelemetry.api.metrics.MeterBuilder;
import io.opentelemetry.api.metrics.MeterProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;

class FlagEvalMetricsLateSdkForkedTest {
@Test
void evaluationBeforeSdkRegistrationDoesNotClaimTheGlobalProvider() {
GlobalOpenTelemetry.resetForTest();
final FlagEvalMetrics metrics = new FlagEvalMetrics();
final FlagEvalMetrics metrics = new FlagEvalMetrics(0);
metrics.record("before", "on", "STATIC", null, null);
assertFalse(GlobalOpenTelemetry.isSet());

Expand All @@ -49,4 +53,56 @@ void evaluationBeforeSdkRegistrationDoesNotClaimTheGlobalProvider() {
GlobalOpenTelemetry.resetForTest();
}
}

@Test
void evaluationsRacingCounterInitializationAreStillRecorded() throws Exception {
GlobalOpenTelemetry.resetForTest();
final CountDownLatch building = new CountDownLatch(1);
final CountDownLatch release = new CountDownLatch(1);
final OpenTelemetry telemetry = mock(OpenTelemetry.class);
final MeterProvider provider = mock(MeterProvider.class);
final MeterBuilder meterBuilder = mock(MeterBuilder.class);
final Meter meter = mock(Meter.class);
final LongCounterBuilder counterBuilder = mock(LongCounterBuilder.class);
final LongCounter counter = mock(LongCounter.class);
when(telemetry.getMeterProvider()).thenReturn(provider);
when(provider.meterBuilder("ddtrace.openfeature"))
.thenAnswer(invocation -> {
building.countDown();
release.await(); // hold the first evaluation inside counter initialization
return meterBuilder;
});
when(meterBuilder.build()).thenReturn(meter);
when(meter.counterBuilder("feature_flag.evaluations")).thenReturn(counterBuilder);
when(counterBuilder.setUnit("{evaluation}")).thenReturn(counterBuilder);
when(counterBuilder.setDescription("Number of feature flag evaluations"))
.thenReturn(counterBuilder);
when(counterBuilder.build()).thenReturn(counter);
GlobalOpenTelemetry.set(telemetry);
// A long recheck interval: a throttled evaluation would be dropped, not retried.
final FlagEvalMetrics metrics = new FlagEvalMetrics(TimeUnit.DAYS.toNanos(1));
try {
final Thread first = new Thread(() -> metrics.record("first", "on", "STATIC", null, null));
first.start();
assertTrue(building.await(10, TimeUnit.SECONDS));
final Thread second = new Thread(() -> metrics.record("second", "on", "STATIC", null, null));
second.start();
// Wait until the second evaluation is blocked behind the first's initialization.
final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
while (second.getState() != Thread.State.BLOCKED && second.isAlive()) {
if (System.nanoTime() > deadline) {
throw new AssertionError("second evaluation never reached the initializer");
}
Thread.sleep(1);
}
release.countDown();
first.join(10_000);
second.join(10_000);
verify(counter, times(2)).add(eq(1L), any(Attributes.class));
} finally {
release.countDown();
metrics.close();
GlobalOpenTelemetry.resetForTest();
}
}
}
Loading