|
3 | 3 |
|
4 | 4 | import numpy as np |
5 | 5 | import pandas as pd |
| 6 | +from scipy import stats |
6 | 7 |
|
7 | 8 | __all__ = ["sensitivity_indices"] |
8 | 9 |
|
|
14 | 15 | HAS_IPYTHON = False |
15 | 16 |
|
16 | 17 |
|
17 | | -def magic_binning(x: np.ndarray, n_bins_default: int) -> tuple[np.ndarray, int]: |
18 | | - """ |
19 | | - Python equivalent of the MATLAB magic_binning function. |
20 | | - Uses equal-frequency binning, groups identical values, and handles NaNs. |
21 | | - """ |
| 18 | +def _quantile_edges(x: np.ndarray, n_bins: int) -> np.ndarray: |
| 19 | + """Bin edges holding approximately the same number of points. |
22 | 20 |
|
23 | | - is_nan = np.isnan(x) |
24 | | - not_nan_x = x[~is_nan] |
| 21 | + Bins are defined by value, so identical values always fall in the same |
| 22 | + bin. Duplicated edges are dropped: a factor with many ties gets fewer |
| 23 | + than ``n_bins`` bins, and a constant factor gets a single one. |
25 | 24 |
|
26 | | - idx_sorted = np.argsort(not_nan_x) |
27 | | - x_sorted = not_nan_x[idx_sorted] |
28 | | - n_valid = len(x_sorted) |
| 25 | + Discrete variables (few unique values relative to n_bins) get one bin |
| 26 | + per unique value instead of quantile-based edges, to avoid collapsing |
| 27 | + minority categories into a majority bin. |
| 28 | + """ |
| 29 | + unique_vals = np.unique(x[~np.isnan(x)]) |
| 30 | + if len(unique_vals) <= n_bins: |
| 31 | + # midpoints between consecutive unique values, so each value gets |
| 32 | + # its own bin |
| 33 | + if unique_vals.size == 1: |
| 34 | + return np.array([unique_vals[0], np.nextafter(unique_vals[0], np.inf)]) |
| 35 | + midpoints = (unique_vals[:-1] + unique_vals[1:]) / 2 |
| 36 | + return np.concatenate([[unique_vals[0]], midpoints, [unique_vals[-1]]]) |
29 | 37 |
|
30 | | - bin_idx_valid = np.zeros(n_valid, dtype=int) |
| 38 | + edges = np.unique(np.nanquantile(x, np.linspace(0, 1, n_bins + 1))) |
| 39 | + if edges.size < 2: |
| 40 | + edges = np.array([edges[0], np.nextafter(edges[0], np.inf)]) |
| 41 | + return edges |
31 | 42 |
|
32 | | - unique_vals = np.unique(x_sorted) |
33 | | - if len(unique_vals) <= n_bins_default: |
34 | | - _, bin_idx_valid = np.unique( |
35 | | - x_sorted, return_inverse=True |
36 | | - ) # x_sorted, not not_nan_x |
37 | | - bin_idx_valid += 1 |
38 | | - else: |
39 | | - min_bin_size = n_valid // n_bins_default |
40 | | - remaining_size = n_valid |
41 | | - current_edge_idx = min_bin_size - 1 # 0-based index |
42 | | - |
43 | | - b = 1 |
44 | | - start_idx = 0 |
45 | | - |
46 | | - while b <= n_bins_default: |
47 | | - current_bin_size = min_bin_size |
48 | | - |
49 | | - # While the edge is between identical values, move one element further |
50 | | - while (current_edge_idx < n_valid - 1) and ( |
51 | | - x_sorted[current_edge_idx + 1] == x_sorted[current_edge_idx] |
52 | | - ): |
53 | | - current_edge_idx += 1 |
54 | | - current_bin_size += 1 |
55 | | - |
56 | | - # Assign bin indices |
57 | | - bin_idx_valid[start_idx : current_edge_idx + 1] = b |
58 | | - remaining_size -= current_bin_size |
59 | | - |
60 | | - # Break if not enough elements left for two distinct bins |
61 | | - if remaining_size < min_bin_size * 2: |
62 | | - bin_idx_valid[current_edge_idx + 1 :] = b + 1 |
63 | | - break |
64 | | - |
65 | | - start_idx = current_edge_idx + 1 |
66 | | - current_edge_idx += min_bin_size |
67 | | - b += 1 |
68 | | - |
69 | | - bin_idx_valid_orig_order = np.zeros(n_valid, dtype=int) |
70 | | - bin_idx_valid_orig_order[idx_sorted] = bin_idx_valid |
71 | | - |
72 | | - # NaNs back in (NaNs get bin 0) |
73 | | - bin_idx = np.zeros(len(x), dtype=int) |
74 | | - bin_idx[~is_nan] = bin_idx_valid_orig_order |
75 | | - |
76 | | - n_bins_out = np.max(bin_idx) if len(bin_idx) > 0 else 0 |
77 | | - |
78 | | - return bin_idx, n_bins_out |
79 | | - |
80 | | - |
81 | | -def bin_data_1d( |
82 | | - x: np.ndarray, y: np.ndarray, n_bins_default: int |
83 | | -) -> tuple[np.ndarray, np.ndarray]: |
84 | | - bin_idx, n_bins_x = magic_binning(x, n_bins_default) |
85 | | - |
86 | | - bin_avg = np.full(n_bins_x, np.nan) |
87 | | - bin_count = np.full(n_bins_x, np.nan) |
88 | | - |
89 | | - for b in range(1, n_bins_x + 1): |
90 | | - mask = bin_idx == b |
91 | | - if np.any(mask): |
92 | | - bin_avg[b - 1] = np.mean(y[mask]) |
93 | | - bin_count[b - 1] = np.sum(mask) |
94 | | - |
95 | | - return bin_avg, bin_count |
96 | | - |
97 | | - |
98 | | -def bin_data_2d(xi: np.ndarray, xj: np.ndarray, y: np.ndarray, n_bins_default: int): |
99 | | - bin_idx_i, n_bins_i = magic_binning(xi, n_bins_default) |
100 | | - bin_idx_j, n_bins_j = magic_binning(xj, n_bins_default) |
101 | | - |
102 | | - bin_avg_ij = np.full((n_bins_i, n_bins_j), np.nan) |
103 | | - bin_count_ij = np.full((n_bins_i, n_bins_j), np.nan) |
104 | | - bin_avg_i = np.full(n_bins_i, np.nan) |
105 | | - bin_count_i = np.full(n_bins_i, np.nan) |
106 | | - bin_avg_j = np.full(n_bins_j, np.nan) |
107 | | - bin_count_j = np.full(n_bins_j, np.nan) |
108 | | - |
109 | | - for n in range(1, n_bins_i + 1): |
110 | | - mask_i = bin_idx_i == n |
111 | | - if np.any(mask_i): |
112 | | - bin_avg_i[n - 1] = np.mean(y[mask_i]) |
113 | | - bin_count_i[n - 1] = np.sum(mask_i) |
114 | | - |
115 | | - for m in range(1, n_bins_j + 1): |
116 | | - mask_j = bin_idx_j == m |
117 | | - if np.any(mask_j): |
118 | | - bin_avg_j[m - 1] = np.mean(y[mask_j]) |
119 | | - bin_count_j[m - 1] = np.sum(mask_j) |
120 | | - |
121 | | - for n in range(1, n_bins_i + 1): |
122 | | - for m in range(1, n_bins_j + 1): |
123 | | - mask_ij = (bin_idx_i == n) & (bin_idx_j == m) |
124 | | - if np.any(mask_ij): |
125 | | - bin_avg_ij[n - 1, m - 1] = np.mean(y[mask_ij]) |
126 | | - bin_count_ij[n - 1, m - 1] = np.sum(mask_ij) |
127 | | - |
128 | | - # Flatten the 2D matrices and exclude NaNs (empty bins) |
129 | | - bin_avg_ij_flat = bin_avg_ij.flatten() |
130 | | - bin_count_ij_flat = bin_count_ij.flatten() |
131 | | - |
132 | | - valid = ~np.isnan(bin_avg_ij_flat) |
133 | | - |
134 | | - return ( |
135 | | - bin_avg_ij_flat[valid], |
136 | | - bin_count_ij_flat[valid], |
137 | | - bin_avg_i, |
138 | | - bin_count_i, |
139 | | - bin_avg_j, |
140 | | - bin_count_j, |
141 | | - ) |
| 43 | + |
| 44 | +def _conditional_var(sample: np.ndarray, y: np.ndarray, edges: list) -> float: |
| 45 | + """Var(E[Y | bins]), each bin weighted by its number of points.""" |
| 46 | + mean, *_ = stats.binned_statistic_dd(sample, y, statistic="mean", bins=edges) |
| 47 | + count, *_ = stats.binned_statistic_dd(sample, y, statistic="count", bins=edges) |
| 48 | + valid = count > 0 |
| 49 | + return _weighted_var(mean[valid], weights=count[valid]) |
142 | 50 |
|
143 | 51 |
|
144 | 52 | def number_of_bins(n_runs: int, n_factors: int) -> tuple[int, int]: |
@@ -264,41 +172,33 @@ def sensitivity_indices( |
264 | 172 | foe = np.empty(n_factors) |
265 | 173 | soe = np.zeros((n_factors, n_factors)) |
266 | 174 |
|
| 175 | + edges_foe = [ |
| 176 | + _quantile_edges(inputs[:, k], int(n_bins_foe)) for k in range(n_factors) |
| 177 | + ] |
| 178 | + edges_soe = [ |
| 179 | + _quantile_edges(inputs[:, k], int(n_bins_soe)) for k in range(n_factors) |
| 180 | + ] |
| 181 | + |
| 182 | + # Marginal Var(E[Y|Xk]) on the SOE binning, identical for every pair |
| 183 | + var_marginal = np.array( |
| 184 | + [ |
| 185 | + _conditional_var(inputs[:, [k]], output, [edges_soe[k]]) |
| 186 | + for k in range(n_factors) |
| 187 | + ] |
| 188 | + ) |
| 189 | + |
267 | 190 | for i in range(n_factors): |
268 | 191 | # 1. First-order effects (FOE) |
269 | | - xi = inputs[:, i] |
| 192 | + foe[i] = _conditional_var(inputs[:, [i]], output, [edges_foe[i]]) / var_y |
270 | 193 |
|
271 | | - bin_avg, bin_count = bin_data_1d(xi, output, int(n_bins_foe)) |
272 | | - |
273 | | - valid_foe = ~np.isnan(bin_avg) |
274 | | - foe[i] = _weighted_var(bin_avg[valid_foe], weights=bin_count[valid_foe]) / var_y |
275 | | - |
276 | | - # 2. Second-order effects (SOE) |
277 | 194 | for j in range(n_factors): |
278 | 195 | if j <= i: |
279 | 196 | continue |
280 | 197 |
|
281 | | - xj = inputs[:, j] |
282 | | - |
283 | | - # Second-order effects (SOE) with magic_binning |
284 | | - ( |
285 | | - bin_avg_ij, |
286 | | - bin_count_ij, |
287 | | - bin_avg_i, |
288 | | - bin_count_i, |
289 | | - bin_avg_j, |
290 | | - bin_count_j, |
291 | | - ) = bin_data_2d(xi, xj, output, int(n_bins_soe)) |
292 | | - |
293 | | - var_ij = _weighted_var(bin_avg_ij, weights=bin_count_ij) |
294 | | - |
295 | | - valid_i = ~np.isnan(bin_avg_i) |
296 | | - var_i = _weighted_var(bin_avg_i[valid_i], weights=bin_count_i[valid_i]) |
297 | | - |
298 | | - valid_j = ~np.isnan(bin_avg_j) |
299 | | - var_j = _weighted_var(bin_avg_j[valid_j], weights=bin_count_j[valid_j]) |
300 | | - |
301 | | - soe[i, j] = (var_ij - var_i - var_j) / var_y |
| 198 | + var_ij = _conditional_var( |
| 199 | + inputs[:, [i, j]], output, [edges_soe[i], edges_soe[j]] |
| 200 | + ) |
| 201 | + soe[i, j] = (var_ij - var_marginal[i] - var_marginal[j]) / var_y |
302 | 202 |
|
303 | 203 | # Mirror SOE and calculate Combined Effect (SI) |
304 | 204 | # SI is FOE + half of all interactions associated with that variable |
|
0 commit comments