Skip to content

Commit aaaf8d9

Browse files
committed
Back to Scipy: Add discrete-variable handling to _quantile_edges.
1 parent 8b80e8d commit aaaf8d9

1 file changed

Lines changed: 49 additions & 149 deletions

File tree

‎src/simdec/sensitivity_indices.py‎

Lines changed: 49 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
import numpy as np
55
import pandas as pd
6+
from scipy import stats
67

78
__all__ = ["sensitivity_indices"]
89

@@ -14,131 +15,38 @@
1415
HAS_IPYTHON = False
1516

1617

17-
def magic_binning(x: np.ndarray, n_bins_default: int) -> tuple[np.ndarray, int]:
18-
"""
19-
Python equivalent of the MATLAB magic_binning function.
20-
Uses equal-frequency binning, groups identical values, and handles NaNs.
21-
"""
18+
def _quantile_edges(x: np.ndarray, n_bins: int) -> np.ndarray:
19+
"""Bin edges holding approximately the same number of points.
2220
23-
is_nan = np.isnan(x)
24-
not_nan_x = x[~is_nan]
21+
Bins are defined by value, so identical values always fall in the same
22+
bin. Duplicated edges are dropped: a factor with many ties gets fewer
23+
than ``n_bins`` bins, and a constant factor gets a single one.
2524
26-
idx_sorted = np.argsort(not_nan_x)
27-
x_sorted = not_nan_x[idx_sorted]
28-
n_valid = len(x_sorted)
25+
Discrete variables (few unique values relative to n_bins) get one bin
26+
per unique value instead of quantile-based edges, to avoid collapsing
27+
minority categories into a majority bin.
28+
"""
29+
unique_vals = np.unique(x[~np.isnan(x)])
30+
if len(unique_vals) <= n_bins:
31+
# midpoints between consecutive unique values, so each value gets
32+
# its own bin
33+
if unique_vals.size == 1:
34+
return np.array([unique_vals[0], np.nextafter(unique_vals[0], np.inf)])
35+
midpoints = (unique_vals[:-1] + unique_vals[1:]) / 2
36+
return np.concatenate([[unique_vals[0]], midpoints, [unique_vals[-1]]])
2937

30-
bin_idx_valid = np.zeros(n_valid, dtype=int)
38+
edges = np.unique(np.nanquantile(x, np.linspace(0, 1, n_bins + 1)))
39+
if edges.size < 2:
40+
edges = np.array([edges[0], np.nextafter(edges[0], np.inf)])
41+
return edges
3142

32-
unique_vals = np.unique(x_sorted)
33-
if len(unique_vals) <= n_bins_default:
34-
_, bin_idx_valid = np.unique(
35-
x_sorted, return_inverse=True
36-
) # x_sorted, not not_nan_x
37-
bin_idx_valid += 1
38-
else:
39-
min_bin_size = n_valid // n_bins_default
40-
remaining_size = n_valid
41-
current_edge_idx = min_bin_size - 1 # 0-based index
42-
43-
b = 1
44-
start_idx = 0
45-
46-
while b <= n_bins_default:
47-
current_bin_size = min_bin_size
48-
49-
# While the edge is between identical values, move one element further
50-
while (current_edge_idx < n_valid - 1) and (
51-
x_sorted[current_edge_idx + 1] == x_sorted[current_edge_idx]
52-
):
53-
current_edge_idx += 1
54-
current_bin_size += 1
55-
56-
# Assign bin indices
57-
bin_idx_valid[start_idx : current_edge_idx + 1] = b
58-
remaining_size -= current_bin_size
59-
60-
# Break if not enough elements left for two distinct bins
61-
if remaining_size < min_bin_size * 2:
62-
bin_idx_valid[current_edge_idx + 1 :] = b + 1
63-
break
64-
65-
start_idx = current_edge_idx + 1
66-
current_edge_idx += min_bin_size
67-
b += 1
68-
69-
bin_idx_valid_orig_order = np.zeros(n_valid, dtype=int)
70-
bin_idx_valid_orig_order[idx_sorted] = bin_idx_valid
71-
72-
# NaNs back in (NaNs get bin 0)
73-
bin_idx = np.zeros(len(x), dtype=int)
74-
bin_idx[~is_nan] = bin_idx_valid_orig_order
75-
76-
n_bins_out = np.max(bin_idx) if len(bin_idx) > 0 else 0
77-
78-
return bin_idx, n_bins_out
79-
80-
81-
def bin_data_1d(
82-
x: np.ndarray, y: np.ndarray, n_bins_default: int
83-
) -> tuple[np.ndarray, np.ndarray]:
84-
bin_idx, n_bins_x = magic_binning(x, n_bins_default)
85-
86-
bin_avg = np.full(n_bins_x, np.nan)
87-
bin_count = np.full(n_bins_x, np.nan)
88-
89-
for b in range(1, n_bins_x + 1):
90-
mask = bin_idx == b
91-
if np.any(mask):
92-
bin_avg[b - 1] = np.mean(y[mask])
93-
bin_count[b - 1] = np.sum(mask)
94-
95-
return bin_avg, bin_count
96-
97-
98-
def bin_data_2d(xi: np.ndarray, xj: np.ndarray, y: np.ndarray, n_bins_default: int):
99-
bin_idx_i, n_bins_i = magic_binning(xi, n_bins_default)
100-
bin_idx_j, n_bins_j = magic_binning(xj, n_bins_default)
101-
102-
bin_avg_ij = np.full((n_bins_i, n_bins_j), np.nan)
103-
bin_count_ij = np.full((n_bins_i, n_bins_j), np.nan)
104-
bin_avg_i = np.full(n_bins_i, np.nan)
105-
bin_count_i = np.full(n_bins_i, np.nan)
106-
bin_avg_j = np.full(n_bins_j, np.nan)
107-
bin_count_j = np.full(n_bins_j, np.nan)
108-
109-
for n in range(1, n_bins_i + 1):
110-
mask_i = bin_idx_i == n
111-
if np.any(mask_i):
112-
bin_avg_i[n - 1] = np.mean(y[mask_i])
113-
bin_count_i[n - 1] = np.sum(mask_i)
114-
115-
for m in range(1, n_bins_j + 1):
116-
mask_j = bin_idx_j == m
117-
if np.any(mask_j):
118-
bin_avg_j[m - 1] = np.mean(y[mask_j])
119-
bin_count_j[m - 1] = np.sum(mask_j)
120-
121-
for n in range(1, n_bins_i + 1):
122-
for m in range(1, n_bins_j + 1):
123-
mask_ij = (bin_idx_i == n) & (bin_idx_j == m)
124-
if np.any(mask_ij):
125-
bin_avg_ij[n - 1, m - 1] = np.mean(y[mask_ij])
126-
bin_count_ij[n - 1, m - 1] = np.sum(mask_ij)
127-
128-
# Flatten the 2D matrices and exclude NaNs (empty bins)
129-
bin_avg_ij_flat = bin_avg_ij.flatten()
130-
bin_count_ij_flat = bin_count_ij.flatten()
131-
132-
valid = ~np.isnan(bin_avg_ij_flat)
133-
134-
return (
135-
bin_avg_ij_flat[valid],
136-
bin_count_ij_flat[valid],
137-
bin_avg_i,
138-
bin_count_i,
139-
bin_avg_j,
140-
bin_count_j,
141-
)
43+
44+
def _conditional_var(sample: np.ndarray, y: np.ndarray, edges: list) -> float:
45+
"""Var(E[Y | bins]), each bin weighted by its number of points."""
46+
mean, *_ = stats.binned_statistic_dd(sample, y, statistic="mean", bins=edges)
47+
count, *_ = stats.binned_statistic_dd(sample, y, statistic="count", bins=edges)
48+
valid = count > 0
49+
return _weighted_var(mean[valid], weights=count[valid])
14250

14351

14452
def number_of_bins(n_runs: int, n_factors: int) -> tuple[int, int]:
@@ -264,41 +172,33 @@ def sensitivity_indices(
264172
foe = np.empty(n_factors)
265173
soe = np.zeros((n_factors, n_factors))
266174

175+
edges_foe = [
176+
_quantile_edges(inputs[:, k], int(n_bins_foe)) for k in range(n_factors)
177+
]
178+
edges_soe = [
179+
_quantile_edges(inputs[:, k], int(n_bins_soe)) for k in range(n_factors)
180+
]
181+
182+
# Marginal Var(E[Y|Xk]) on the SOE binning, identical for every pair
183+
var_marginal = np.array(
184+
[
185+
_conditional_var(inputs[:, [k]], output, [edges_soe[k]])
186+
for k in range(n_factors)
187+
]
188+
)
189+
267190
for i in range(n_factors):
268191
# 1. First-order effects (FOE)
269-
xi = inputs[:, i]
192+
foe[i] = _conditional_var(inputs[:, [i]], output, [edges_foe[i]]) / var_y
270193

271-
bin_avg, bin_count = bin_data_1d(xi, output, int(n_bins_foe))
272-
273-
valid_foe = ~np.isnan(bin_avg)
274-
foe[i] = _weighted_var(bin_avg[valid_foe], weights=bin_count[valid_foe]) / var_y
275-
276-
# 2. Second-order effects (SOE)
277194
for j in range(n_factors):
278195
if j <= i:
279196
continue
280197

281-
xj = inputs[:, j]
282-
283-
# Second-order effects (SOE) with magic_binning
284-
(
285-
bin_avg_ij,
286-
bin_count_ij,
287-
bin_avg_i,
288-
bin_count_i,
289-
bin_avg_j,
290-
bin_count_j,
291-
) = bin_data_2d(xi, xj, output, int(n_bins_soe))
292-
293-
var_ij = _weighted_var(bin_avg_ij, weights=bin_count_ij)
294-
295-
valid_i = ~np.isnan(bin_avg_i)
296-
var_i = _weighted_var(bin_avg_i[valid_i], weights=bin_count_i[valid_i])
297-
298-
valid_j = ~np.isnan(bin_avg_j)
299-
var_j = _weighted_var(bin_avg_j[valid_j], weights=bin_count_j[valid_j])
300-
301-
soe[i, j] = (var_ij - var_i - var_j) / var_y
198+
var_ij = _conditional_var(
199+
inputs[:, [i, j]], output, [edges_soe[i], edges_soe[j]]
200+
)
201+
soe[i, j] = (var_ij - var_marginal[i] - var_marginal[j]) / var_y
302202

303203
# Mirror SOE and calculate Combined Effect (SI)
304204
# SI is FOE + half of all interactions associated with that variable

0 commit comments

Comments
 (0)